internal static IIngressStreamable <Empty, TPayload> ToCheckpointableStreamable <TPayload>(
this IObservable <TPayload> streamEvents,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy completedPolicy,
TimelinePolicy timelinePolicy,
QueryContainer container,
string identifier)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
return(new MonotonicIngressStreamable <TPayload>(
streamEvents,
flushPolicy,
periodicPunctuationPolicy,
completedPolicy,
timelinePolicy ?? TimelinePolicy.WallClock(),
container,
identifier));
}
public MonotonicSubscriptionWallClock(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
if (timelinePolicy.punctuationInterval > default(TimeSpan))
{
this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval);
}
}
public MonotonicIngressStreamable(
IObservable <TPayload> observable,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy completedPolicy,
TimelinePolicy timelinePolicy,
QueryContainer container,
string identifier)
: base((Config.ForceRowBasedExecution || !typeof(TPayload).CanRepresentAsColumnar()
? StreamProperties <Empty, TPayload> .Default.ToRowBased()
: StreamProperties <Empty, TPayload> .Default).ToConstantDuration(true, StreamEvent.InfinitySyncTime).SetQueryContainer(container))
{
Contract.Requires(observable != null);
Contract.Requires(identifier != null);
this.IngressSiteIdentifier = identifier;
this.observable = observable;
this.flushPolicy = flushPolicy;
this.punctuationPolicy = punctuationPolicy;
this.onCompletedPolicy = completedPolicy;
this.timelinePolicy = timelinePolicy;
this.container = container;
this.delayed = container != null;
if (this.delayed)
{
container.RegisterIngressSite(identifier);
}
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableStreamable <TPayload>(
this IObservable <StreamEvent <TPayload> > streamEvents,
QueryContainer container,
string identifier,
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (disorderPolicy == null)
{
disorderPolicy = DisorderPolicy.Throw();
}
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
var a = new StreamEventIngressStreamable <TPayload>(
streamEvents,
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
container,
identifier);
return(a);
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableTemporalStreamable <TPayload>(
this IObservable <TPayload> streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
QueryContainer container,
string identifier,
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (disorderPolicy == null)
{
disorderPolicy = DisorderPolicy.Throw();
}
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
var a = new IntervalIngressStreamable <TPayload>(
streamEvents,
startEdgeExtractor,
(p) => StreamEvent.InfinitySyncTime,
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
container,
identifier);
return(a);
}
internal FlushTestBase(
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy completedPolicy) : base(baseConfigModifier)
{
this.disorderPolicy = disorderPolicy;
this.flushPolicy = flushPolicy;
this.punctuationPolicy = punctuationPolicy;
this.completedPolicy = completedPolicy;
}
/// <summary>
/// Registers a data source to be used as input with the query container.
/// </summary>
/// <typeparam name="TPayload">The type of the payload of the data source.</typeparam>
/// <param name="data">A sequence of data elements created by the client.</param>
/// <param name="timelinePolicy">Describes how to simulate a progression of time with the data.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
public static IIngressStreamable <Empty, TPayload> ToAtemporalStreamable <TPayload>(
this IObservable <TPayload> data,
TimelinePolicy timelinePolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(data.ToCheckpointableStreamable(
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
timelinePolicy,
null,
Guid.NewGuid().ToString()));
}
/// <summary>
/// Converts a sequence of StreamEvents to an IStreamable.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="disorderPolicy">How to handle events that are not in time order.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if the <paramref name="disorderPolicy"/> is to throw and
/// an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> ToStreamable <TPayload>(
this IObservable <StreamEvent <TPayload> > streamEvents,
DisorderPolicy disorderPolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(ToCheckpointableStreamable(
streamEvents,
null,
Guid.NewGuid().ToString(),
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy));
}
/// <summary>
/// Registers a data source to be used as input with the query container.
/// </summary>
/// <typeparam name="TPayload">The type of the payload of the data source.</typeparam>
/// <param name="data">A sequence of data elements created by the client.</param>
/// <param name="timelinePolicy">Describes how to simulate a progression of time with the data.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="container">A query containter to which this ingress point can be attached.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
public static IIngressStreamable <Empty, TPayload> RegisterAtemporalInput <TPayload>(
this QueryContainer container,
IObservable <TPayload> data,
TimelinePolicy timelinePolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(data.ToCheckpointableStreamable(
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
timelinePolicy,
container,
identifier ?? Guid.NewGuid().ToString()));
}
/// <summary>
/// Converts a sequence of StreamEvents to an IStreamable.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <param name="container">The query container to which to register the ingress point.</param>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="disorderPolicy">How to handle events that are not in time order.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if the <paramref name="disorderPolicy"/> is to throw and
/// an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> RegisterInput <TPayload>(
this QueryContainer container,
IObservable <StreamEvent <TPayload> > streamEvents,
DisorderPolicy disorderPolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(ToCheckpointableStreamable(
streamEvents,
container,
identifier ?? Guid.NewGuid().ToString(),
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy));
}
/// <summary>
/// Converts a sequence of data elements to an IStreamable, start-edge only.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="startEdgeExtractor">An expresion that describes how to interpret the start time for each data value.</param>
/// <param name="disorderPolicy">How to handle events that are not in time order.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if the <paramref name="disorderPolicy"/> is to throw and
/// an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> ToTemporalStreamable <TPayload>(
this IObservable <TPayload> streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
DisorderPolicy disorderPolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(ToCheckpointableTemporalStreamable(
streamEvents,
startEdgeExtractor,
null,
Guid.NewGuid().ToString(),
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy));
}
/// <summary>
/// Converts a sequence of data elements to an IStreamable, intervals and start-edges only.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <param name="container">The query container to which to register the ingress point.</param>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="startEdgeExtractor">An expresion that describes how to interpret the start time for each data value.</param>
/// <param name="endEdgeExtractor">An expresion that describes how to interpret the end time for each data value. Return StreamEvent.InfinitySyncTime to indicate an event with no end time.</param>
/// <param name="disorderPolicy">How to handle events that are not in time order.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if the <paramref name="disorderPolicy"/> is to throw and
/// an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> RegisterTemporalInput <TPayload>(
this QueryContainer container,
IObservable <TPayload> streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
DisorderPolicy disorderPolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(ToCheckpointableTemporalStreamable(
streamEvents,
startEdgeExtractor,
endEdgeExtractor,
container,
identifier ?? Guid.NewGuid().ToString(),
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy));
}
public MonotonicSubscriptionSequence(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
this.eventsPerSample = timelinePolicy.sampleSize;
this.currentTime = 0;
}
/// <summary>
/// Builder method for <see cref="IndexWriterConfig.FlushPolicy"/>.
/// </summary>
/// <param name="config">this <see cref="IndexWriterConfig"/> instance</param>
/// <param name="flushPolicy"></param>
/// <returns>this <see cref="IndexWriterConfig"/> instance</returns>
internal static IndexWriterConfig SetFlushPolicy(this IndexWriterConfig config, FlushPolicy flushPolicy)
{
config.FlushPolicy = flushPolicy;
return(config);
}
/// <summary>
/// Initializes a new instance of the <see cref="Logger"/> class.
/// </summary>
/// <param name="logLevel">Defines which types of messages to display and which not.</param>
/// <param name="flushPolicy">Defines the way of displaying messages.</param>
public Logger(LogLevel logLevel, FlushPolicy flushPolicy)
{
_level = logLevel;
_flushPolicy = flushPolicy;
}
