/// <summary>
/// Writes the envelopes for the specified stream to a multi-stream \psi store.
/// </summary>
/// <typeparam name="TIn">The type of messages in the stream.</typeparam>
/// <param name="source">The source stream for which to write envelopes.</param>
/// <param name="name">The name of the persisted stream.</param>
/// <param name="writer">The store writer, created by e.g. <see cref="PsiStore.Create(Pipeline, string, string, bool, KnownSerializers)"/>.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The input stream.</returns>
public static IProducer <TIn> WriteEnvelopes <TIn>(this IProducer <TIn> source, string name, Exporter writer, DeliveryPolicy <TIn> deliveryPolicy = null)
{
writer.WriteEnvelopes(source.Out, name, deliveryPolicy);
return(source);
}
/// <summary>
/// Executes a transform function for each item in the input stream, generating a new stream with the values returned by the function.
/// The function has access to the envelope of the input message.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="selector">The function to perform on every message in the source stream. The function takes two parameters, the input message and its envelope.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut> Select <TIn, TOut>(this IProducer <TIn> source, Func <TIn, Envelope, TOut> selector, DeliveryPolicy deliveryPolicy = null)
{
return(Process <TIn, TOut>(source, (d, e, s) => s.Post(selector(d, e), e.OriginatingTime), deliveryPolicy));
}
/// <summary>
/// Executes a transform function for each non-null item in the input stream, generating a new stream with the values returned by the function, or null if the input was null.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="selector">The function to perform on every message in the source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut?> NullableSelect <TIn, TOut>(this IProducer <TIn?> source, Func <TIn, TOut> selector, DeliveryPolicy deliveryPolicy = null)
where TIn : struct
where TOut : struct
{
return(source.Select(v => v.HasValue ? new TOut?(selector(v.Value)) : null, deliveryPolicy));
}
/// <summary>
/// Map messages to their originating time.
/// </summary>
/// <typeparam name="T">Type of source stream messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Stream of originating times.</returns>
public static IProducer <DateTime> TimeOf <T>(this IProducer <T> source, DeliveryPolicy deliveryPolicy = null)
{
return(source.Select((_, e) => e.OriginatingTime, deliveryPolicy));
}
/// <summary>
/// Delay messages by given time span.
/// </summary>
/// <typeparam name="T">Type of source/output messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="delay">Time span by which to delay.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> Delay <T>(this IProducer <T> source, TimeSpan delay, DeliveryPolicy deliveryPolicy = null)
{
return(source
.Process <T, (T, DateTime)>((d, e, s) => s.Post((d, e.OriginatingTime), e.OriginatingTime + delay), deliveryPolicy)
.Process <(T, DateTime), T>((t, _, s) => s.Post(t.Item1, t.Item2), DeliveryPolicy.Unlimited));
}
/// <summary>
/// Filter stream to the first n messages.
/// </summary>
/// <typeparam name="T">Type of source/output messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="number">Number of messages.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> First <T>(this IProducer <T> source, int number, DeliveryPolicy <T> deliveryPolicy = null)
{
return(source.Where(v => number-- > 0, deliveryPolicy));
}
/// <summary>
/// Allows a receiver to subscribe to messages from this emitter.
/// </summary>
/// <param name="receiver">The receiver subscribing to this emitter.</param>
/// <param name="allowSubscribeWhileRunning"> If true, bypasses checks that subscriptions are not made while pipelines are running.</param>
/// <param name="deliveryPolicy">The desired policy to use when delivering messages to the specified receiver.</param>
internal void Subscribe(Receiver <T> receiver, bool allowSubscribeWhileRunning, DeliveryPolicy <T> deliveryPolicy)
{
receiver.OnSubscribe(this, allowSubscribeWhileRunning, deliveryPolicy);
lock (this.receivers)
{
var newSet = this.receivers.Concat(new[] { receiver }).ToArray();
this.receivers = newSet;
}
}
/// <summary>
/// Writes the specified stream to a multi-stream store.
/// </summary>
/// <typeparam name="TIn">The type of messages in the stream</typeparam>
/// <param name="source">The source stream to write</param>
/// <param name="name">The name of the persisted stream.</param>
/// <param name="writer">The store writer, created by e.g. <see cref="Store.Create(Pipeline, string, string, bool, KnownSerializers)"/></param>
/// <param name="largeMessages">Indicates whether the stream contains large messages (typically >4k). If true, the messages will be written to the large message file.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The input stream</returns>
public static IProducer <TIn> Write <TIn>(this IProducer <TIn> source, string name, Exporter writer, bool largeMessages = false, DeliveryPolicy deliveryPolicy = null)
{
writer.Write(source.Out, name, largeMessages, deliveryPolicy);
return(source);
}
/// <summary>
/// Executes a function for each item in the input stream, generating a new stream with the (zero or more) values returned by the function.
/// The function must return an <see cref="IEnumerable{TOut}"/>, which can be <see cref="System.Linq.Enumerable.Empty{TOut}"/> to indicate zero results.
/// The values in the returned <see cref="IEnumerable{TOut}"/> are emitted as separate messages with the same oringinating time.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="selector">The function to perform on every message in the source stream. The function has access to the envelope of the input message.</param>
/// <param name="policy">An optional delivery policy</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut> SelectMany <TIn, TOut>(this IProducer <TIn> source, Func <TIn, Envelope, IEnumerable <TOut> > selector, DeliveryPolicy policy = null)
{
var p = Process <TIn, TOut>(
source,
(d, e, s) =>
{
foreach (var item in selector(d, e))
{
s.Post(item, e.OriginatingTime);
}
},
policy);
return(p);
}
/// <summary>
/// Serializes the source stream, preserving the envelope.
/// </summary>
/// <typeparam name="T">The type of data to serialize</typeparam>
/// <param name="source">The source stream generating the data to serialize</param>
/// <param name="serializers">An optional collection of known types to use</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of Message{Message{BufferReader}}, where the inner Message object preserves the original envelope of the message received by this operator.</returns>
public static IProducer <Message <BufferReader> > Serialize <T>(this IProducer <T> source, KnownSerializers serializers = null, DeliveryPolicy deliveryPolicy = null)
{
var serializer = new SerializerComponent <T>(source.Out.Pipeline, serializers ?? KnownSerializers.Default);
source.PipeTo(serializer.In, deliveryPolicy);
return(serializer.Out);
}
/// <summary>
/// Deserializes data from a stream of Message{BufferReader}.
/// </summary>
/// <typeparam name="T">The type of data expected after deserialization</typeparam>
/// <param name="source">The stream containing the serialized data</param>
/// <param name="serializers">An optional collection of known types to use</param>
/// <param name="reusableInstance">An optional preallocated instance ot use as a buffer. This parameter is required when deserializing <see cref="Shared{T}"/> instances if the deserializer is expected to use a <see cref="SharedPool{T}"/></param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of messages of type T, with their original envelope</returns>
public static IProducer <T> Deserialize <T>(this IProducer <Message <BufferReader> > source, KnownSerializers serializers = null, T reusableInstance = default(T), DeliveryPolicy deliveryPolicy = null)
{
var deserializer = new DeserializerComponent <T>(source.Out.Pipeline, serializers ?? KnownSerializers.Default, reusableInstance);
source.PipeTo(deserializer, deliveryPolicy);
return(deserializer.Out);
}
/// <summary>
/// Publishes the specified stream to the debug partition, allowing debugging visualizers to display the data.
/// </summary>
/// <typeparam name="T">The type of data in the stream.</typeparam>
/// <param name="source">The stream to visualize.</param>
/// <param name="name">The name to use when visualizing the stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The debug name of the stream, either as provided or the generated one if one was not specified.</returns>
public static string DebugView <T>(this IProducer <T> source, string name = null, DeliveryPolicy <T> deliveryPolicy = null)
{
var debugName = name ?? source.Out.Name ?? source.Out.Id.ToString();
if (debugStore != null)
{
lock (syncRoot)
{
if (!PsiStore.TryGetStreamMetadata(debugPipeline, debugName, out IStreamMetadata meta))
{
source.Write(debugName, debugStore, deliveryPolicy: deliveryPolicy);
}
}
}
return(debugName);
}
/// <summary>
/// Zip one or more streams (T) into a single stream (Message{T}) while ensuring delivery in originating time order (ordered within single tick by stream ID).
/// </summary>
/// <remarks>Messages are produced in originating-time order; potentially delayed in wall-clock time.</remarks>
/// <typeparam name="T">Type of messages.</typeparam>
/// <param name="inputs">Collection of homogeneous inputs.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Stream of zipped messages.</returns>
public static IProducer <Message <T> > Zip <T>(IEnumerable <IProducer <T> > inputs, DeliveryPolicy <T> deliveryPolicy = null)
{
if (inputs.Count() == 0)
{
throw new ArgumentException("Zip requires one or more inputs.");
}
var zip = new Zip <T>(inputs.First().Out.Pipeline);
foreach (var i in inputs)
{
i.PipeTo(zip.AddInput($"Receiver{i.Out.Id}"), deliveryPolicy);
}
return(zip.Out);
}
/// <summary>
/// Convert a stream to an <see cref="IObservable{T}"/>.
/// </summary>
/// <typeparam name="T">Type of messages for the source stream.</typeparam>
/// <param name="stream">The source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Observable with elements from the source stream.</returns>
public static IObservable <T> ToObservable <T>(this IProducer <T> stream, DeliveryPolicy deliveryPolicy = null)
{
return(new StreamObservable <T>(stream, deliveryPolicy));
}
/// <summary>
/// Filter messages to those where a given condition is met.
/// </summary>
/// <typeparam name="T">Type of source/output messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="condition">Predicate function by which to filter messages.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> Where <T>(this IProducer <T> source, Func <T, Envelope, bool> condition, DeliveryPolicy <T> deliveryPolicy = null)
{
return(Process <T, T>(
source,
(d, e, s) =>
{
if (condition(d, e))
{
s.Post(d, e.OriginatingTime);
}
},
deliveryPolicy));
}
/// <summary>
/// Executes a function for each item in the input stream, generating a new stream with the (zero or more) values returned by the function.
/// The function must return an <see cref="IEnumerable{TOut}"/>, which can be <see cref="System.Linq.Enumerable.Empty{TOut}"/> to indicate zero results.
/// The values in the returned <see cref="IEnumerable{TOut}"/> are emitted as separate messages with the same oringinating time.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="selector">The function to perform on every message in the source stream.</param>
/// <param name="policy">An optional delivery policy</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut> SelectMany <TIn, TOut>(this IProducer <TIn> source, Func <TIn, IEnumerable <TOut> > selector, DeliveryPolicy policy = null)
{
return(SelectMany(source, (d, e) => selector(d), policy));
}
/// <summary>
/// Filter messages to those where a given condition is met.
/// </summary>
/// <typeparam name="T">Type of source/output messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="condition">Predicate function by which to filter messages.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> Where <T>(this IProducer <T> source, Predicate <T> condition, DeliveryPolicy <T> deliveryPolicy = null)
{
return(Where(source, (d, e) => condition(d), deliveryPolicy));
}
/// <summary>
/// Connnects a stream producer to a stream consumer. As a result, all messages in the stream will be routed to the consumer for processing.
/// </summary>
/// <typeparam name="TIn">The type of messages in the stream.</typeparam>
/// <typeparam name="TC">The type of consumer</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="consumer">The consumer (subscriber)</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The consumer (subscriber).</returns>
public static TC PipeTo <TIn, TC>(this IProducer <TIn> source, TC consumer, DeliveryPolicy deliveryPolicy = null)
where TC : IConsumer <TIn>
{
return(PipeTo(source, consumer, false, deliveryPolicy));
}
/// <summary>
/// Filter stream to the first message (single-message stream).
/// </summary>
/// <typeparam name="T">Type of source/output messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> First <T>(this IProducer <T> source, DeliveryPolicy <T> deliveryPolicy = null)
{
return(First(source, 1, deliveryPolicy));
}
/// <summary>
/// Connnects a stream producer to a stream consumer. As a result, all messages in the stream will be routed to the consumer for processing.
/// </summary>
/// <remarks>
/// This is an internal-only method which provides the option to allow connections between producers and consumers in running pipelines.
/// </remarks>
/// <typeparam name="TIn">The type of messages in the stream.</typeparam>
/// <typeparam name="TC">The type of consumer</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="consumer">The consumer (subscriber)</param>
/// <param name="allowWhileRunning">An optional flag to allow connections in running pipelines.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The consumer (subscriber).</returns>
internal static TC PipeTo <TIn, TC>(this IProducer <TIn> source, TC consumer, bool allowWhileRunning, DeliveryPolicy deliveryPolicy = null)
where TC : IConsumer <TIn>
{
source.Out.Subscribe(consumer.In, allowWhileRunning, deliveryPolicy ?? source.Out.Pipeline.DeliveryPolicy);
return(consumer);
}
/// <summary>
/// Convert a stream to an <see cref="IObservable{T}"/>.
/// </summary>
/// <typeparam name="T">Type of messages for the source stream.</typeparam>
/// <param name="stream">The source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <param name="name">An optional name for this stream operator.</param>
/// <returns>Observable with elements from the source stream.</returns>
public static IObservable <T> ToObservable <T>(this IProducer <T> stream, DeliveryPolicy <T> deliveryPolicy = null, string name = nameof(ToObservable))
=> new StreamObservable <T>(stream, deliveryPolicy, name);
/// <summary>
/// Repeat last seen source message (or default or initial value) upon clock signal.
/// </summary>
/// <typeparam name="T">Type of stream messages.</typeparam>
/// <typeparam name="TClock">Type of clock signal.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="clock">Clock signal stream.</param>
/// <param name="useInitialValue">Whether to seed with an initial value (before any messages seen).</param>
/// <param name="initialValue">Initial value (repeated before any messages seen).</param>
/// <param name="policy">Delivery policy.</param>
/// <returns>Output stream.</returns>
public static IProducer <T> Repeat <T, TClock>(this IProducer <T> source, IProducer <TClock> clock, bool useInitialValue = false, T initialValue = default(T), DeliveryPolicy policy = null)
{
policy = policy ?? DeliveryPolicy.Throttled;
var repeater = new Repeater <T, TClock>(source.Out.Pipeline, useInitialValue, initialValue);
clock.PipeTo(repeater.ClockIn, policy);
source.PipeTo(repeater.In, policy);
return(repeater);
}
/// <summary>
/// Map messages to their current latency (time since origination).
/// </summary>
/// <typeparam name="T">Type of source stream messages.</typeparam>
/// <param name="source">Source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Stream of latency (time span) values.</returns>
public static IProducer <TimeSpan> Latency <T>(this IProducer <T> source, DeliveryPolicy deliveryPolicy = null)
{
return(source.Select((_, e) => e.Time - e.OriginatingTime, deliveryPolicy));
}
/// <summary>
/// Creates a typed delivery policy with guarantees by adding a message guaranteed function to an existing untyped delivery policy.
/// </summary>
/// <typeparam name="T">The type of the messages in the resulting delivery policy.</typeparam>
/// <param name="guaranteeDelivery">A function that evaluates whether the delivery of a given message should be guaranteed.</param>
/// <returns>The typed delivery policy with guarantees.</returns>
public DeliveryPolicy <T> WithGuarantees <T>(Func <T, bool> guaranteeDelivery)
{
return(DeliveryPolicy.WithGuarantees(this, guaranteeDelivery));
}
/// <summary>
/// Executes a transform action for each item in the input stream. The action can output zero or more results by posting them to the emitter provided as an argument.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="transform">The action to perform on every message in the source stream.
/// The action parameters are the message, the envelope and an emitter to post results to</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut> Process <TIn, TOut>(this IProducer <TIn> source, Action <TIn, Envelope, Emitter <TOut> > transform, DeliveryPolicy deliveryPolicy = null)
{
var select = new Processor <TIn, TOut>(source.Out.Pipeline, transform);
return(PipeTo(source, select, deliveryPolicy));
}
/// <summary>
/// Create pipeline.
/// </summary>
/// <param name="name">Pipeline name.</param>
/// <param name="globalPolicy">Global delivery policy.</param>
/// <param name="threadCount">Number of threads.</param>
/// <param name="allowSchedulingOnExternalThreads">Whether to allow scheduling on external threads.</param>
/// <returns>Created pipeline.</returns>
public static Pipeline Create(string name = null, DeliveryPolicy globalPolicy = null, int threadCount = 0, bool allowSchedulingOnExternalThreads = false)
{
return(new Pipeline(name, globalPolicy, threadCount, allowSchedulingOnExternalThreads));
}
/// <summary>
/// Executes a transform function for each item in the input stream, generating a new stream with the values returned by the function.
/// </summary>
/// <typeparam name="TIn">The input message type</typeparam>
/// <typeparam name="TOut">The output message type</typeparam>
/// <param name="source">The source stream to subscribe to</param>
/// <param name="selector">The function to perform on every message in the source stream.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>A stream of type <typeparamref name="TOut"/></returns>
public static IProducer <TOut> Select <TIn, TOut>(this IProducer <TIn> source, Func <TIn, TOut> selector, DeliveryPolicy deliveryPolicy = null)
{
return(Select(source, (d, e) => selector(d), deliveryPolicy));
}
/// <summary>
/// Zip two streams (T) into a single stream while ensuring delivery in originating time order.
/// </summary>
/// <remarks>Messages are produced in originating-time order; potentially delayed in wall-clock time.
/// If multiple messages arrive with the same originating time, they are added in the output array in
/// the order of stream ids.</remarks>
/// <typeparam name="T">Type of messages.</typeparam>
/// <param name="input1">First input stream.</param>
/// <param name="input2">Second input stream with same message type.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>Stream of zipped messages.</returns>
public static IProducer <T[]> Zip <T>(this IProducer <T> input1, IProducer <T> input2, DeliveryPolicy <T> deliveryPolicy = null)
{
return(Zip(new List <IProducer <T> >()
{
input1, input2
}, deliveryPolicy));
}
internal void OnSubscribe(Emitter <T> source, bool allowSubscribeWhileRunning, DeliveryPolicy <T> policy)
{
if (this.Source != null)
{
throw new InvalidOperationException("This receiver is already connected to a source emitter.");
}
if (!allowSubscribeWhileRunning && (this.pipeline.IsRunning || source.Pipeline.IsRunning))
{
throw new InvalidOperationException("Attempting to connect a receiver to an emitter while pipeline is already running. Make all connections before running the pipeline.");
}
if (source.Pipeline != this.pipeline)
{
throw new InvalidOperationException("Receiver cannot subscribe to an emitter from a different pipeline. Use a Connector if you need to connect emitters and receivers from different pipelines.");
}
this.Source = source;
this.DeliveryPolicy = policy;
this.awaitingDelivery = new DeliveryQueue <T>(policy, this.Recycler);
this.pipeline.DiagnosticsCollector?.PipelineElementReceiverSubscribe(this.pipeline, this.element, this, source, this.DeliveryPolicy.Name);
}
/// <summary>
/// Writes the specified stream to a multi-stream \psi store.
/// </summary>
/// <typeparam name="TMessage">The type of messages in the stream.</typeparam>
/// <typeparam name="TSupplementalMetadata">The type of supplemental stream metadata.</typeparam>
/// <param name="source">The source stream to write.</param>
/// <param name="supplementalMetadataValue">Supplemental metadata value.</param>
/// <param name="name">The name of the persisted stream.</param>
/// <param name="writer">The store writer, created by e.g. <see cref="PsiStore.Create(Pipeline, string, string, bool, KnownSerializers)"/>.</param>
/// <param name="largeMessages">Indicates whether the stream contains large messages (typically >4k). If true, the messages will be written to the large message file.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
/// <returns>The input stream.</returns>
public static IProducer <TMessage> Write <TMessage, TSupplementalMetadata>(this IProducer <TMessage> source, TSupplementalMetadata supplementalMetadataValue, string name, Exporter writer, bool largeMessages = false, DeliveryPolicy <TMessage> deliveryPolicy = null)
{
writer.Write(source.Out, supplementalMetadataValue, name, largeMessages, deliveryPolicy);
return(source);
}
