public MonotonicSubscriptionWallClock(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
if (timelinePolicy.punctuationInterval > default(TimeSpan))
{
this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval);
}
}
public MonotonicIngressStreamable(
IObservable <TPayload> observable,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy completedPolicy,
TimelinePolicy timelinePolicy,
QueryContainer container,
string identifier)
: base((Config.ForceRowBasedExecution || !typeof(TPayload).CanRepresentAsColumnar()
? StreamProperties <Empty, TPayload> .Default.ToRowBased()
: StreamProperties <Empty, TPayload> .Default).ToConstantDuration(true, StreamEvent.InfinitySyncTime).SetQueryContainer(container))
{
Contract.Requires(observable != null);
Contract.Requires(identifier != null);
this.IngressSiteIdentifier = identifier;
this.observable = observable;
this.flushPolicy = flushPolicy;
this.punctuationPolicy = punctuationPolicy;
this.onCompletedPolicy = completedPolicy;
this.timelinePolicy = timelinePolicy;
this.container = container;
this.delayed = container != null;
if (this.delayed)
{
container.RegisterIngressSite(identifier);
}
}
internal static IIngressStreamable <Empty, TPayload> ToCheckpointableStreamable <TPayload>(
this IObservable <TPayload> streamEvents,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy completedPolicy,
TimelinePolicy timelinePolicy,
QueryContainer container,
string identifier)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
return(new MonotonicIngressStreamable <TPayload>(
streamEvents,
flushPolicy,
periodicPunctuationPolicy,
completedPolicy,
timelinePolicy ?? TimelinePolicy.WallClock(),
container,
identifier));
}
public IntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
public PartitionedIntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, TKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.partitionExtractor = partitionExtractor;
this.partitionFunction = partitionExtractor.Compile();
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
static void Main(string[] args)
{
// Poll performance counter 4 times a second
TimeSpan pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source = new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e)) // Create an IObservable of StreamEvent<>
.ToStreamable(OnCompletedPolicy.Flush(), PeriodicPunctuationPolicy.Count(4)); // Create a streamable with a punctuation every 4 events
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream.AlterEventDuration(windowSize).Average(e => e.Value);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
static void Main(string[] args)
{
// ingress data
var input =
Observable
.Range(0, 100)
.Select(e => new MyStruct {
field1 = e % 10, field2 = e + 0.5, field3 = "blah"
})
.Select(e => StreamEvent.CreateStart(StreamEvent.MinSyncTime, e))
.ToStreamable(OnCompletedPolicy.EndOfStream(), PeriodicPunctuationPolicy.None())
.Cache();
// query
var query =
input.SetProperty().IsConstantDuration(true)
.GroupBy(e => e.field2)
.SelectMany(str => str.Distinct(f => f.field1), (g, c) => c);
// egress data
query
.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine("{0}\t{1}\t{2}", e.SyncTime, e.OtherTime, e.Payload)).Wait();
Console.ReadLine();
}
public PartitionedIntervalArrayIngressStreamable(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, TPartitionKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
OnCompletedPolicy onCompletedPolicy,
QueryContainer container,
string identifier)
: base((Config.ForceRowBasedExecution || !typeof(TPayload).CanRepresentAsColumnar()
? StreamProperties <PartitionKey <TPartitionKey>, TPayload> .DefaultIngress(startEdgeExtractor, endEdgeExtractor).ToRowBased()
: StreamProperties <PartitionKey <TPartitionKey>, TPayload> .DefaultIngress(startEdgeExtractor, endEdgeExtractor)).SetQueryContainer(container))
{
Contract.Requires(observable != null);
Contract.Requires(identifier != null);
this.IngressSiteIdentifier = identifier;
this.observable = observable;
this.partitionExtractor = partitionExtractor;
this.startEdgeExtractor = startEdgeExtractor;
this.endEdgeExtractor = endEdgeExtractor;
this.onCompletedPolicy = onCompletedPolicy;
this.container = container;
this.delayed = container != null;
if (delayed)
{
container.RegisterIngressSite(identifier);
}
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableStreamable <TPayload>(
this IObservable <StreamEvent <TPayload> > streamEvents,
QueryContainer container,
string identifier,
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (disorderPolicy == null)
{
disorderPolicy = DisorderPolicy.Throw();
}
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
var a = new StreamEventIngressStreamable <TPayload>(
streamEvents,
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
container,
identifier);
return(a);
}
/// <summary>
/// Converts a sequence of data elements to an IStreamable, intervals and start-edges only, with a partition key.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <param name="container">The query container to which to register the ingress point.</param>
/// <typeparam name="TPartitionKey">The type of partition key for the stream.</typeparam>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="partitionExtractor">An expresion that describes how to interpret the partition identifier for each data value.</param>
/// <param name="startEdgeExtractor">An expresion that describes how to interpret the start time for each data value.</param>
/// <param name="endEdgeExtractor">An expresion that describes how to interpret the end time for each data value. Return StreamEvent.InfinitySyncTime to indicate an event with no end time.</param>
/// <param name="disorderPolicy">How to handle events that are not in time order.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="periodicLowWatermarkPolicy">Whether to add periodic low watermarks to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if the <paramref name="disorderPolicy"/> is to throw and
/// an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IPartitionedIngressStreamable <TPartitionKey, TPayload> RegisterPartitionedInput <TPartitionKey, TPayload>(
this QueryContainer container,
IObservable <TPayload> streamEvents,
Expression <Func <TPayload, TPartitionKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
DisorderPolicy disorderPolicy = null,
PartitionedFlushPolicy flushPolicy = PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
PeriodicLowWatermarkPolicy periodicLowWatermarkPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(ToCheckpointablePartitionedStreamable(
streamEvents,
partitionExtractor,
startEdgeExtractor,
endEdgeExtractor,
container,
identifier ?? Guid.NewGuid().ToString(),
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
periodicLowWatermarkPolicy,
onCompletedPolicy));
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableTemporalStreamable <TPayload>(
this IObservable <TPayload> streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
QueryContainer container,
string identifier,
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy periodicPunctuationPolicy,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
if (disorderPolicy == null)
{
disorderPolicy = DisorderPolicy.Throw();
}
if (periodicPunctuationPolicy == null)
{
periodicPunctuationPolicy = PeriodicPunctuationPolicy.None();
}
var a = new IntervalIngressStreamable <TPayload>(
streamEvents,
startEdgeExtractor,
(p) => StreamEvent.InfinitySyncTime,
disorderPolicy,
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
container,
identifier);
return(a);
}
public static PartitionedObserverSubscriptionBase <TKey, ArraySegment <PartitionedStreamEvent <TKey, TPayload> >, TPayload, TPayload> CreateSubscription(
IObservable <ArraySegment <PartitionedStreamEvent <TKey, TPayload> > > observable,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
{
return(new PartitionedStreamEventArraySubscriptionThrowNone <TKey, TPayload>(observable, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
}
/// <summary>
/// Converts a sequence of StreamEvents to an IStreamable.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> ToStreamable <TPayload>(
this IObservable <ArraySegment <StreamEvent <TPayload> > > streamEvents,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(ToCheckpointableStreamable(
streamEvents,
null,
Guid.NewGuid().ToString(),
onCompletedPolicy));
}
/// <summary>
/// Registers a data source to be used as input with the query container.
/// </summary>
/// <typeparam name="TPayload">The type of the payload of the data source.</typeparam>
/// <param name="data">A sequence of data elements created by the client.</param>
/// <param name="timelinePolicy">Describes how to simulate a progression of time with the data.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
public static IIngressStreamable <Empty, TPayload> ToAtemporalArrayStreamable <TPayload>(
this IObservable <ArraySegment <TPayload> > data,
TimelinePolicy timelinePolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.None)
{
return(data.ToCheckpointableStreamable(
onCompletedPolicy,
timelinePolicy,
null,
Guid.NewGuid().ToString()));
}
private static IStreamable <Empty, SensorReading> CreateStream(bool isRealTime)
{
if (isRealTime)
{
return(SimulateLiveData()
.Select(r => StreamEvent.CreateInterval(r.Time, r.Time + 1, r))
.ToStreamable(OnCompletedPolicy.EndOfStream(), PeriodicPunctuationPolicy.Count(1)));
}
return(HistoricData.ToObservable().Select(r => StreamEvent.CreateInterval(r.Time, r.Time + 1, r)).ToStreamable(OnCompletedPolicy.None()));
}
internal FlushTestBase(
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy completedPolicy) : base(baseConfigModifier)
{
this.disorderPolicy = disorderPolicy;
this.flushPolicy = flushPolicy;
this.punctuationPolicy = punctuationPolicy;
this.completedPolicy = completedPolicy;
}
/// <summary>
/// Converts a sequence of data elements to an IStreamable, start-edge only.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="startEdgeExtractor">An expresion that describes how to interpret the start time for each data value.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> ToTemporalArrayStreamable <TPayload>(
this IObservable <ArraySegment <TPayload> > streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(ToCheckpointableTemporalArrayStreamable(
streamEvents,
startEdgeExtractor,
null,
Guid.NewGuid().ToString(),
onCompletedPolicy));
}
/// <summary>
/// Converts a sequence of StreamEvents to an IStreamable.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <param name="container">The query container to which to register the ingress point.</param>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> RegisterInput <TPayload>(
this QueryContainer container,
IObservable <ArraySegment <StreamEvent <TPayload> > > streamEvents,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(ToCheckpointableStreamable(
streamEvents,
container,
identifier ?? Guid.NewGuid().ToString(),
onCompletedPolicy));
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableStreamable <TPayload>(
this IObservable <ArraySegment <StreamEvent <TPayload> > > streamEvents,
QueryContainer container,
string identifier,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
var a = new StreamEventArrayIngressStreamable <TPayload>(streamEvents, onCompletedPolicy, container, identifier);
return(a);
}
public static ObserverSubscriptionBase <ArraySegment <TPayload>, TPayload, TPayload> CreateSubscription(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
{
return(new IntervalArraySubscriptionThrowNone <TPayload>(observable, startEdgeExtractor, endEdgeExtractor, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
}
private static void ArrayBasedIngressExample()
{
// Create the first array segment.
StreamEvent <Point>[] values1 =
{
StreamEvent.CreateInterval(1, 10, new Point {
x = 1, y = 2
}),
StreamEvent.CreateInterval(2, 10, new Point {
x = 2, y = 4
}),
StreamEvent.CreateInterval(3, 10, new Point {
x = 3, y = 6
}),
StreamEvent.CreateInterval(4, 10, new Point {
x = 4, y = 8
}),
StreamEvent.CreateInterval(5, 10, new Point {
x = 5, y = 10
})
};
// Create the second array segment.
StreamEvent <Point>[] values2 =
{
StreamEvent.CreateInterval(6, 10, new Point {
x = 6, y = 12
}),
StreamEvent.CreateInterval(7, 10, new Point {
x = 7, y = 14
}),
StreamEvent.CreateInterval(8, 10, new Point {
x = 8, y = 16
}),
StreamEvent.CreateInterval(9, 10, new Point {
x = 9, y = 18
}),
StreamEvent.CreatePunctuation <Point>(StreamEvent.InfinitySyncTime)
};
var segment1 = new ArraySegment <StreamEvent <Point> >(values1);
var segment2 = new ArraySegment <StreamEvent <Point> >(values2);
var segments = new ArraySegment <StreamEvent <Point> >[] { segment1, segment2 };
// Array-based ingress.
var input = segments.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
Console.ReadLine();
}
internal static IObservableIngressStreamable <TPayload> ToCheckpointableTemporalArrayStreamable <TPayload>(
this IObservable <ArraySegment <TPayload> > streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
QueryContainer container,
string identifier,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
var a = new IntervalArrayIngressStreamable <TPayload>(streamEvents, startEdgeExtractor, (o) => StreamEvent.InfinitySyncTime, onCompletedPolicy, container, identifier);
return(a);
}
/// <summary>
/// Registers a data source to be used as input with the query container.
/// </summary>
/// <typeparam name="TPayload">The type of the payload of the data source.</typeparam>
/// <param name="data">A sequence of data elements created by the client.</param>
/// <param name="timelinePolicy">Describes how to simulate a progression of time with the data.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="container">A query containter to which this ingress point can be attached.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
public static IIngressStreamable <Empty, TPayload> RegisterAtemporalArrayInput <TPayload>(
this QueryContainer container,
IObservable <ArraySegment <TPayload> > data,
TimelinePolicy timelinePolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.None,
string identifier = null)
{
return(data.ToCheckpointableStreamable(
onCompletedPolicy,
timelinePolicy,
container,
identifier ?? Guid.NewGuid().ToString()));
}
/// <summary>
/// Converts a sequence of data elements to an IStreamable, start-edge only.
/// The completion policy specifies what to do when the resulting stream completes.
/// The disorder policy specifies what to do with out of order events.
/// The punctuation policy specifies whether and how punctuations are created and injected
/// into the resulting stream. Since punctuations force output, this exposes a throughput/latency tradeoff.
/// </summary>
/// <param name="container">The query container to which to register the ingress point.</param>
/// <typeparam name="TPayload">The type of data for the stream.</typeparam>
/// <param name="streamEvents">A sequence of stream events created by the client.</param>
/// <param name="startEdgeExtractor">An expresion that describes how to interpret the start time for each data value.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <param name="identifier">If provided, a unique name to identify to point of ingress in the query.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
/// <exception cref="IngressException">
/// Throws an exception if an out-of-order stream event is encountered.
/// Also, an exception is thrown if any payload is null.
/// </exception>
public static IObservableIngressStreamable <TPayload> RegisterTemporalArrayInput <TPayload>(
this QueryContainer container,
IObservable <ArraySegment <TPayload> > streamEvents,
Expression <Func <TPayload, long> > startEdgeExtractor,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream,
string identifier = null)
{
return(ToCheckpointableTemporalArrayStreamable(
streamEvents,
startEdgeExtractor,
container,
identifier ?? Guid.NewGuid().ToString(),
onCompletedPolicy));
}
static void CreateData()
{
source1 = new StreamEvent <Payload>[] {
StreamEvent.CreatePoint(100, new Payload {
Field1 = "A", Field2 = 4
}),
StreamEvent.CreatePoint(110, new Payload {
Field1 = "C", Field2 = 3
}),
StreamEvent.CreatePoint(120, new Payload {
Field1 = "A", Field2 = 1
}),
StreamEvent.CreatePoint(130, new Payload {
Field1 = "B", Field2 = 6
}),
StreamEvent.CreatePoint(140, new Payload {
Field1 = "B", Field2 = 8
}),
StreamEvent.CreatePoint(150, new Payload {
Field1 = "C", Field2 = 7
}),
StreamEvent.CreatePoint(160, new Payload {
Field1 = "B", Field2 = 9
}),
}.ToObservable().ToStreamable(OnCompletedPolicy.EndOfStream()).AlterEventDuration(1000).Cache();
source2 = new StreamEvent <Payload>[] {
StreamEvent.CreatePoint(100, new Payload {
Field1 = "A", Field2 = 1
}),
StreamEvent.CreatePoint(110, new Payload {
Field1 = "C", Field2 = 2
}),
StreamEvent.CreatePoint(120, new Payload {
Field1 = "A", Field2 = 2
}),
StreamEvent.CreatePoint(130, new Payload {
Field1 = "B", Field2 = 2
}),
StreamEvent.CreatePoint(140, new Payload {
Field1 = "B", Field2 = 2
}),
StreamEvent.CreatePoint(150, new Payload {
Field1 = "C", Field2 = 1
}),
StreamEvent.CreatePoint(160, new Payload {
Field1 = "B", Field2 = 1
}),
}.ToObservable().ToStreamable(OnCompletedPolicy.EndOfStream()).AlterEventDuration(1000).Cache();
}
internal static IPartitionedIngressStreamable <TPartitionKey, TPayload> ToCheckpointablePartitionedArrayStreamable <TPartitionKey, TPayload>(
this IObservable <ArraySegment <TPayload> > streamEvents,
Expression <Func <TPayload, TPartitionKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
QueryContainer container,
string identifier,
OnCompletedPolicy onCompletedPolicy)
{
Contract.EnsuresOnThrow <IngressException>(true);
var a = new PartitionedIntervalArrayIngressStreamable <TPartitionKey, TPayload>(streamEvents, partitionExtractor, startEdgeExtractor, endEdgeExtractor, onCompletedPolicy, container, identifier);
return(a);
}
private static void SelectFunc()
{
var input = values.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
Console.WriteLine();
Console.WriteLine("Query: input.Select(r => new { Original = r, Squared = r * r, Text = \"Hello #\" + r })");
var output = input.Select(r => new { Original = r, Squared = r * r, Text = "Hello #" + r });
Console.WriteLine();
Console.WriteLine("Output =");
output.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
}
private static void AlterLifetimeFunc()
{
var input = values.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
Console.WriteLine();
Console.WriteLine("Query: input.AlterEventLifetime(oldStart => oldStart + 5, 2)");
var output = input.AlterEventLifetime(oldStart => oldStart + 5, 2);
Console.WriteLine();
Console.WriteLine("Output =");
output.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
}
private static void SelectManyFunc()
{
var input = values.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
Console.WriteLine();
Console.WriteLine("Query: input.SelectMany(r => Enumerable.Repeat(r, 5))");
var output = input.SelectMany(r => Enumerable.Repeat(r, 5));
Console.WriteLine();
Console.WriteLine("Output =");
output.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
}
/// <summary>
/// Registers a data source to be used as input with the query container.
/// </summary>
/// <typeparam name="TPayload">The type of the payload of the data source.</typeparam>
/// <param name="data">A sequence of data elements created by the client.</param>
/// <param name="timelinePolicy">Describes how to simulate a progression of time with the data.</param>
/// <param name="flushPolicy">When to flush batched output events.</param>
/// <param name="periodicPunctuationPolicy">Whether to add periodic punctuations to the resulting stream.</param>
/// <param name="onCompletedPolicy">How to handle the completion of a stream.</param>
/// <returns>An IStreamable that can be used in queries.</returns>
public static IIngressStreamable <Empty, TPayload> ToAtemporalStreamable <TPayload>(
this IObservable <TPayload> data,
TimelinePolicy timelinePolicy = null,
FlushPolicy flushPolicy = FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy periodicPunctuationPolicy = null,
OnCompletedPolicy onCompletedPolicy = OnCompletedPolicy.EndOfStream)
{
return(data.ToCheckpointableStreamable(
flushPolicy,
periodicPunctuationPolicy,
onCompletedPolicy,
timelinePolicy,
null,
Guid.NewGuid().ToString()));
}
