static void Main(string[] args)
{
// Poll performance counter 4 times a second
TimeSpan pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source = new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e)) // Create an IObservable of StreamEvent<>
.ToStreamable(OnCompletedPolicy.Flush(), PeriodicPunctuationPolicy.Count(4)); // Create a streamable with a punctuation every 4 events
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream.AlterEventDuration(windowSize).Average(e => e.Value);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
public static void Main(string[] args)
{
// Poll performance counter 4 times a second
var pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source =
new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill, injecting a punctuation every second. Because we use
// FlushPolicy.FlushOnPunctuation, this will also flush the data every second.
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e))
.ToStreamable(
null,
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromSeconds(1).Ticks));
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream.AlterEventDuration(windowSize).Average(e => e.Value);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
static void Main(string[] args)
{
// ingress data
var input =
Observable
.Range(0, 100)
.Select(e => new MyStruct {
field1 = e % 10, field2 = e + 0.5, field3 = "blah"
})
.Select(e => StreamEvent.CreateStart(StreamEvent.MinSyncTime, e))
.ToStreamable(OnCompletedPolicy.EndOfStream(), PeriodicPunctuationPolicy.None())
.Cache();
// query
var query =
input.SetProperty().IsConstantDuration(true)
.GroupBy(e => e.field2)
.SelectMany(str => str.Distinct(f => f.field1), (g, c) => c);
// egress data
query
.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine("{0}\t{1}\t{2}", e.SyncTime, e.OtherTime, e.Payload)).Wait();
Console.ReadLine();
}
static void Main(string[] args)
{
// create the data source
// note: we cannot create more than one of these right now (limitation on the unmanaged side for simplicity)
IObservable <NativeData> source = new NativeObservable();
// create the stream processor
var inputStream = source
.Select(e => StreamEvent.CreateStart(DateTime.Now.Ticks, e))
.ToStreamable(
null,
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromSeconds(1).Ticks));
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream
.AlterEventDuration(windowSize)
// we "filter" using our unmanaged matcher
.Where(e => NativeSubscription.MatchData(e.Data))
.Select(e => e.Data);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
internal TriPartitionedOrderedTestsBase(
ConfigModifier config,
DisorderPolicy disorderPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
PeriodicLowWatermarkPolicy lowWatermarkPolicy) : base(config)
{
this.disorderPolicy = disorderPolicy;
this.punctuationPolicy = punctuationPolicy;
this.lowWatermarkPolicy = lowWatermarkPolicy;
}
internal FlushTestBase(
DisorderPolicy disorderPolicy,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy completedPolicy) : base(baseConfigModifier)
{
this.disorderPolicy = disorderPolicy;
this.flushPolicy = flushPolicy;
this.punctuationPolicy = punctuationPolicy;
this.completedPolicy = completedPolicy;
}
private static IStreamable <Empty, SensorReading> CreateStream(bool isRealTime)
{
if (isRealTime)
{
return(SimulateLiveData()
.Select(r => StreamEvent.CreateInterval(r.Time, r.Time + 1, r))
.ToStreamable(OnCompletedPolicy.EndOfStream(), PeriodicPunctuationPolicy.Count(1)));
}
return(HistoricData.ToObservable().Select(r => StreamEvent.CreateInterval(r.Time, r.Time + 1, r)).ToStreamable(OnCompletedPolicy.None()));
}
public static IStreamable <Empty, T> ToStatStreamable <T>(this IEnumerable <T> source)
{
var batchSize = 80000;
var timeStamps = StreamEvent.MinSyncTime.CountToInfinity().SelectMany(i => Enumerable.Repeat(i, batchSize));
var streamEventEnumerable = Enumerable.Zip(timeStamps, source, (t, e) => StreamEvent.CreateStart(t, e))
.Concat(new StreamEvent <T>[] { StreamEvent.CreatePunctuation <T>(StreamEvent.InfinitySyncTime), });
var stream = streamEventEnumerable
.ToObservable(Scheduler.Default)
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.Time(1))
.SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime);
return(stream);
}
public void AlterLifeTimeWithFlushDoubles()
{
var startDate = new DateTime(100);
var data = new[]
{
new { Time = startDate, DeviceId = 4, Value = 4 },
new { Time = startDate, DeviceId = 4, Value = 4 },
new { Time = startDate, DeviceId = 6, Value = 6 },
new { Time = startDate, DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(1), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(1), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(2), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(2), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(3), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(3), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(11), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(11), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(5), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(5), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(12), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(12), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(13), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(13), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(14), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(14), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(15), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(15), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(16), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(16), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(17), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(17), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(21), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(21), DeviceId = 6, Value = 6 },
}
.Select(e => PartitionedStreamEvent.CreateInterval(e.DeviceId, e.Time.Ticks, StreamEvent.MaxSyncTime, e.Value));
var res = new List <Notification <PartitionedStreamEvent <int, int> > >();
using (data.ToObservable()
.ToStreamable(
DisorderPolicy.Adjust(0),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.Time(36000))
.AlterEventDuration(1)
.ToStreamEventObservable()
.Subscribe(res.NotificationListObserver()))
{
}
Assert.IsTrue(res.Count > 0, "There should be some results.");
}
/// <summary>
/// Create query and register subscriber
/// </summary>
private void CreateQuery()
{
this.queryContainer = new QueryContainer();
this.input = new Subject <StreamEvent <long> >();
var inputStream = this.queryContainer.RegisterInput(
this.input,
DisorderPolicy.Drop(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time(1));
var query = inputStream.AlterEventDuration(StreamEvent.InfinitySyncTime).Count();
var async = this.queryContainer.RegisterOutput(query);
async.Where(e => e.IsStart).ForEachAsync(o => Console.WriteLine($"{o}"));
}
public static void Main(string[] args)
{
// Poll performance counter 4 times a second
var pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source =
new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill, injecting a punctuation every second. Because we use
// FlushPolicy.FlushOnPunctuation, this will also flush the data every second.
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e))
.ToStreamable(
null,
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromSeconds(1).Ticks));
// Query 1: Aggregate query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query1 = inputStream.TumblingWindowLifetime(windowSize).Average(e => e.Value);
// Query 2: look for pattern of [CPU < 10] --> [CPU >= 10]
var query2 = inputStream
.AlterEventDuration(TimeSpan.FromSeconds(10).Ticks)
.Detect(default(Tuple <float, float>), // register to store CPU value
p => p
.SingleElement(e => e.Value < 10, (ts, ev, reg) => new Tuple <float, float>(ev.Value, 0))
.SingleElement(e => e.Value >= 10, (ts, ev, reg) => new Tuple <float, float>(reg.Item1, ev.Value)));
// Query 3: look for pattern of [k increasing CPU values --> drop CPU], report max, k
int k = 2;
var query3 = inputStream
.AlterEventDuration(TimeSpan.FromSeconds(10).Ticks)
.Detect(default(Tuple <float, int>), // register to store CPU value, incr count
p => p
.KleenePlus(e =>
e.SingleElement((ev, reg) => reg == null || ev.Value > reg.Item1, (ts, ev, reg) => new Tuple <float, int>(ev.Value, reg == null ? 1 : reg.Item2 + 1)))
.SingleElement((ev, reg) => ev.Value <reg.Item1 && reg.Item2> k, (ts, ev, reg) => reg),
allowOverlappingInstances: false);
// Egress results and write to console
query3.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
private QueryContainer CreateQuery()
{
var query = new QueryContainer();
Config.ForceRowBasedExecution = true;
// incoming events are received via a conduit
_tweetConduit = new Subject <StreamEvent <Tweet> >();
var streamInput = query.RegisterInput(_tweetConduit, OnCompletedPolicy.None(), DisorderPolicy.Drop(TimeSpan.FromSeconds(3).Ticks),
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromMilliseconds(1).Ticks));
// outgoing events are pushed to the dashboard
var myOutput = query.RegisterOutput(TwitterAnalytics(streamInput), ReshapingPolicy.None());
myOutput.Subscribe(tweetEvent => Publish(tweetEvent));
return(query);
}
private void SetupQuery(
DisorderPolicy disorderPolicy,
PartitionedFlushPolicy flushPolicy = PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy punctuationPolicy = null, // Default: PeriodicPunctuationPolicy.None()
PeriodicLowWatermarkPolicy lowWatermarkPolicy = null, // Default: PeriodicLowWatermarkPolicy.None()
OnCompletedPolicy completedPolicy = OnCompletedPolicy.EndOfStream)
{
var qc = new QueryContainer();
this.input = new Subject <PartitionedStreamEvent <int, int> >();
var ingress = qc.RegisterInput(this.input, disorderPolicy, flushPolicy, punctuationPolicy,
lowWatermarkPolicy, completedPolicy);
this.output = new List <PartitionedStreamEvent <int, int> >();
this.egress = qc.RegisterOutput(ingress).ForEachAsync(o => this.output.Add(o));
this.process = qc.Restore();
this.validateByPartition = disorderPolicy.reorderLatency > 0;
}
public void PunctuationTimeGeneration()
{
SetupQuery(
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.Time(generationPeriod: 1000));
// This establishes three partitions with keys 1,2,3
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// Since the next point arrives at 10001, which is exactly PeriodicPunctuationPolicy.generationPeriod after
// the first point, this will generate a punctuation at 10001
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 10001, 0));
// These are still valid, since the punctuation generated above only affects key 0
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 1, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 1, 2));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Flushed in response to OnCompleted
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
// Punctuation snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 10000),
PartitionedStreamEvent.CreatePoint(0, 10001, 0),
PartitionedStreamEvent.CreatePoint(1, 1, 1),
PartitionedStreamEvent.CreatePoint(2, 1, 2),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
private void ProcessInput(
out List <OutOfOrderPartitionedStreamEvent <int, int> > diagnosticEvents,
out List <PartitionedStreamEvent <int, int> > dataEvents)
{
var qc = new QueryContainer();
var ingress = qc.RegisterInput(this.input.ToObservable(), this.disorderPolicy,
PartitionedFlushPolicy.None, PeriodicPunctuationPolicy.None(), this.lowWatermarkPolicy);
var outOfOrderEvents = new List <OutOfOrderPartitionedStreamEvent <int, int> >();
ingress.GetDroppedAdjustedEventsDiagnostic().Subscribe(o => outOfOrderEvents.Add(o));
var output = new List <PartitionedStreamEvent <int, int> >();
var egress = qc.RegisterOutput(ingress).ForEachAsync(o => output.Add(o));
var process = qc.Restore();
process.Flush();
egress.Wait();
diagnosticEvents = outOfOrderEvents;
dataEvents = output;
}
private static void PassThrough()
{
// var inputStream = GetTollReadings();
// Display(inputStream);
// var subscription = TollReadingEvents.ToAtemporalStreamable();
// Display(subscription);
{
var queryContainer = new QueryContainer();
var inputStream = queryContainer.RegisterInput(
TollReadingEvents,
DisorderPolicy.Drop(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time(1));
var query = inputStream.HoppingWindowLifetime(TimeSpan.FromSeconds(10).Ticks, TimeSpan.FromSeconds(1).Ticks)
.Count()
.AlterEventDuration(1);
var async = queryContainer.RegisterOutput(query);
Display("1) ", 1, async);
queryProcess1 = queryContainer.Restore();
}
}
private static IObservable <StreamEvent <Toll> > GetTollObservable(QueryContainer queryContainer)
{
var inputStream = queryContainer.RegisterInput(TollReadingEvents,
DisorderPolicy.Drop(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time(1));
var partitionedSlidingWindow1 =
inputStream.AlterEventDuration((start, end) => end - start + TimeSpan.FromSeconds(2).Ticks)
.HoppingWindowLifetime(TimeSpan.FromSeconds(2).Ticks, TimeSpan.FromSeconds(2).Ticks)
.GroupApply(
r => r.TollId,
r =>
r.Multicast(
perTollBoth =>
perTollBoth.Sum(e => e.Toll)
.Join(perTollBoth.Count(), (sum, count) => new { Sum = sum, Count = count })),
(key, result) =>
new Toll {
TollId = key.Key, TollAmount = result.Sum, VehicleCount = result.Count
});
var retval = queryContainer.RegisterOutput(partitionedSlidingWindow1);
return(retval);
}
public FlushOnBatchBoundary_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnBatchBoundary, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
{
}
public FlushNone_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.None, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
{
}
public FlushOnPunctuation_OnCompletedFlush() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.Flush)
{
}
public static void Test()
{
// This program is a supplement to the Trill Users Guide
#region Section 2
IObservable <ContextSwitch> cSTicksObs = new[]
{
new ContextSwitch(0, 1, 1, 120),
new ContextSwitch(0, 3, 2, 121),
new ContextSwitch(0, 5, 3, 124),
new ContextSwitch(120, 2, 1, 123),
new ContextSwitch(300, 1, 1, 122),
new ContextSwitch(1800, 4, 2, 125),
new ContextSwitch(3540, 2, 1, 119),
new ContextSwitch(3600, 1, 1, 120),
}.ToObservable();
Console.WriteLine("Figure 11: Complete Pass-Through Trill Query Program");
var cSTicksEventObs = cSTicksObs.Select(
e => StreamEvent.CreateInterval(e.CSTicks, e.CSTicks + 1, e));
var cSTicksStream =
cSTicksEventObs.ToStreamable(DisorderPolicy.Drop());
var origCSTicksEventObs = cSTicksStream.ToStreamEventObservable();
origCSTicksEventObs.Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tCPUTemp={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID,
e.Payload.CPUTemp)).Wait();
Console.WriteLine();
#endregion
#region Section 3
Console.WriteLine("Figure 13: Where Query Code");
var cSTicks2Cores = cSTicksStream.Where(p => p.CID == 1 || p.CID == 2);
cSTicks2Cores.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tCPUTemp={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID,
e.Payload.CPUTemp)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 16: Select Query Code");
var cSNarrowTicks2Cores = cSTicks2Cores.Select(
e => new { CSTicks = e.CSTicks, PID = e.PID, CID = e.CID });
cSNarrowTicks2Cores.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 17: Alternate Select Query Code");
var cSNarrowTicks2Cores2 = cSTicks2Cores.Select(
e => new { CSTicks = e.CSTicks, PID = e.PID, CID = e.CID });
cSNarrowTicks2Cores2.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 19: Where and Select with the LinQ Comprehension Syntax");
var cSNarrowTicks2Cores3 = from e in cSTicksStream
where e.CID == 1 || e.CID == 2
select new { e.CSTicks, e.PID, e.CID };
cSNarrowTicks2Cores3.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID)).Wait();
Console.WriteLine();
#endregion
#region Section 4
Console.WriteLine("Figure 22: Join Query Code");
var processNamesObs = new[]
{
new ProcessName(1, "Word"),
new ProcessName(2, "Internet Explorer"),
new ProcessName(3, "Excel"),
new ProcessName(4, "Visual Studio"),
new ProcessName(5, "Outlook"),
}.ToObservable();
var pNamesStream = processNamesObs.
Select(e => StreamEvent.CreateInterval(0, 10000, e)).
ToStreamable();
var cSTicks2CoresWithPNames = cSNarrowTicks2Cores.Join(pNamesStream, e => e.PID, e => e.PID,
(leftPayload, rightPayload) => new
{
leftPayload.CSTicks,
leftPayload.PID,
leftPayload.CID,
PName = rightPayload.PName
});
cSTicks2CoresWithPNames.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 24: Join Query Comprehension Syntax");
var cSTicks2CoresWithPNames2 = from leftPayload in cSNarrowTicks2Cores
join rightPayload in pNamesStream on
leftPayload.PID equals rightPayload.PID
select new
{
leftPayload.CSTicks,
leftPayload.PID,
leftPayload.CID,
PName = rightPayload.PName
};
cSTicks2CoresWithPNames2.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 25: Entire Join Query Comprehension Syntax");
var cSTicks2CoresWithPNames3 = from leftPayload in cSTicksStream
join rightPayload in pNamesStream on
leftPayload.PID equals rightPayload.PID
where leftPayload.CID == 1 || leftPayload.CID == 2
select new
{
leftPayload.CSTicks,
leftPayload.PID,
leftPayload.CID,
rightPayload.PName
};
cSTicks2CoresWithPNames3.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
#endregion
#region Section 5
Console.WriteLine("Figure 26: AlterEventDuration Query Code");
var infinitecSTicks2Cores =
cSTicks2CoresWithPNames.AlterEventDuration(StreamEvent.InfinitySyncTime);
infinitecSTicks2Cores.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
#endregion Section 6
#region Section 6
Console.WriteLine("Figure 28: ClipEventDuration Query Code");
var clippedCSTicks2Cores = infinitecSTicks2Cores.
ClipEventDuration(infinitecSTicks2Cores, e => e.CID, e => e.CID);
clippedCSTicks2Cores.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
#endregion
#region Section 7
Console.WriteLine("Figure 30: Multicast Version of ClipEventDuration Query Code");
var clippedCSTicks2Cores2 = infinitecSTicks2Cores.Multicast(
s => s.ClipEventDuration(s, e => e.CID, e => e.CID));
clippedCSTicks2Cores2.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
#endregion
#region Section 8
Console.WriteLine("Figure 32: ShiftEventLifetime Query Code");
var shiftedClippedCS = clippedCSTicks2Cores.ShiftEventLifetime(startTime => 1);
shiftedClippedCS.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks={2}\tPID={3}\tCID={4}\tPName={5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.PName)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 34: Timeslices Query Code");
var timeslices = shiftedClippedCS.Join(cSTicks2CoresWithPNames, e => e.CID, e => e.CID,
(left, right) => new
{
left.PID,
left.CID,
left.PName,
Timeslice = right.CSTicks - left.CSTicks
});
timeslices.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tPID={2}\tCID={3}\tPName={4}\tTimeslice={5}",
e.StartTime, e.EndTime, e.Payload.PID, e.Payload.CID, e.Payload.PName, e.Payload.Timeslice)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 36: Timeslices Query Code Using Multicast");
var timeslices2 = cSTicks2CoresWithPNames.Multicast(
t => t.AlterEventDuration(StreamEvent.InfinitySyncTime).
Multicast(s => s.ClipEventDuration(s, e => e.CID, e => e.CID)).
ShiftEventLifetime(startTime => 1).
Join(t, e => e.CID, e => e.CID, (left, right) => new
{
left.PID,
left.CID,
left.PName,
Timeslice = right.CSTicks - left.CSTicks
}));
timeslices2.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tPID={2}\tCID={3}\tPName={4}\tTimeslice={5}",
e.StartTime, e.EndTime, e.Payload.PID, e.Payload.CID, e.Payload.PName, e.Payload.Timeslice)).Wait();
Console.WriteLine();
#endregion
#region Section 9
Console.WriteLine("Figure 40: Rassigning Timeslice Lifetimes with AlterEventLifetime");
var pID1CID1Timeslices = timeslices.Where(e => e.PID == 1 && e.CID == 1);
var windowedTimeslices = pID1CID1Timeslices.
AlterEventLifetime(origStartTime => (1 + (origStartTime - 1) / 3600) * 3600, 3600);
windowedTimeslices.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tPID={2}\tCID={3}\tPName={4}\tTimeslice={5}",
e.StartTime, e.EndTime, e.Payload.PID, e.Payload.CID, e.Payload.PName, e.Payload.Timeslice)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 41: Reassigning Lifetimes with HoppingWindowLifetime");
var windowedTimeslices2 = pID1CID1Timeslices.HoppingWindowLifetime(3600, 3600);
windowedTimeslices2.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tPID={2}\tCID={3}\tPName={4}\tTimeslice={5}",
e.StartTime, e.EndTime, e.Payload.PID, e.Payload.CID, e.Payload.PName, e.Payload.Timeslice)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 42: Sum Query Code");
var totalConsumptionPerPeriod = windowedTimeslices.Sum(e => e.Timeslice);
totalConsumptionPerPeriod.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tPayload={2}",
e.StartTime, e.EndTime, e.Payload)).Wait();
Console.WriteLine();
#endregion
#region Section 10
Console.WriteLine("Figure 44: Group and Apply Query Code");
var totalConsumptionPerPeriodAllCoresAllProcesses = timeslices.GroupApply(
e => new { e.CID, e.PID, e.PName },
s => s.HoppingWindowLifetime(3600, 3600).Sum(e => e.Timeslice),
(g, p) => new { g.Key.CID, g.Key.PName, TotalTime = p });
totalConsumptionPerPeriodAllCoresAllProcesses.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCID={2}\tPName={3}\tTotaltime={4}",
e.StartTime, e.EndTime, e.Payload.CID, e.Payload.PName, e.Payload.TotalTime)).Wait();
Console.WriteLine();
#endregion
#region Section 11
Console.WriteLine("Figure 49: Chop Query Code");
Console.WriteLine("NB: This query is expected to fail!");
var altCSTicksObs = new[]
{
new ContextSwitch(0, 1, 1, 120),
new ContextSwitch(0, 3, 2, 121),
new ContextSwitch(0, 5, 3, 124),
new ContextSwitch(120, 2, 1, 123),
new ContextSwitch(300, 1, 1, 122),
new ContextSwitch(1800, 4, 2, 125),
new ContextSwitch(3540, 2, 1, 119),
new ContextSwitch(3600, 1, 1, 120),
new ContextSwitch(5400, 3, 2, 122),
new ContextSwitch(7200, 4, 2, 121),
}.ToObservable();
var altCSTicksEventObs = altCSTicksObs.Select(
e => StreamEvent.CreateInterval(e.CSTicks, e.CSTicks + 1, e));
var altCSTicksStream =
altCSTicksEventObs.ToStreamable(DisorderPolicy.Drop());
var cSTicksWithExtraCS = altCSTicksStream.AlterEventDuration(StreamEvent.InfinitySyncTime).
Multicast(s => s.ClipEventDuration(s, e => e.CID, e => e.CID)).
Chop(0, 3600).
Select((origStartTime, e) =>
new { CSTicks = origStartTime, e.PID, e.CID, e.CPUTemp }).
AlterEventDuration(1);
try
{
cSTicksWithExtraCS.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks{2}\tPID={3}\tCID={4}\tCPUTemp{5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.CPUTemp)).Wait();
}
catch (AggregateException e)
{
foreach (var ex in e.InnerExceptions)
{
Console.WriteLine("{0}", ex.Message);
}
}
catch (Exception e)
{
Console.WriteLine("{0}", e.Message);
}
Console.WriteLine();
Console.WriteLine("Figure 51: Improved Chop Query Code");
var cSTicksWithExtraCSImp = altCSTicksStream.
AlterEventDuration(StreamEvent.InfinitySyncTime).
Multicast(s => s.ClipEventDuration(s, e => e.CID, e => e.CID)).
Join(new[] { StreamEvent.CreateInterval(0, 10800, Unit.Default) }.
ToObservable().ToStreamable(),
(left, right) => left).Chop(0, 3600).Select((origStartTime, e) =>
new { e.CID, e.PID, e.CPUTemp, CSTicks = origStartTime }).
AlterEventDuration(1);
cSTicksWithExtraCSImp.ToStreamEventObservable().Where(e => e.IsData).ForEachAsync(e => Console.WriteLine(
"Start Time={0}\tEnd Time={1}\tCSTicks{2}\tPID={3}\tCID={4}\tCPUTemp{5}",
e.StartTime, e.EndTime, e.Payload.CSTicks, e.Payload.PID, e.Payload.CID, e.Payload.CPUTemp)).Wait();
Console.WriteLine();
#endregion
#region Section 12
Console.WriteLine("Figure 53: Out of Order Input (Throw)");
var ooStreamableThrow = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Throw());
try
{
ooStreamableThrow.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
}
catch (Exception e)
{
Console.WriteLine("{0}", e.Message);
}
Console.WriteLine();
Console.WriteLine("Figure 54: Out of Order Input (Drop)");
var ooStreamableDrop = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Drop());
ooStreamableDrop.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 55: Out of Order Input (Adjust)");
var ooStreamableAdjust = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Adjust());
ooStreamableAdjust.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 57: Creating PNamesStream with Edge Events");
var pNamesStream2 = processNamesObs.Select(e => StreamEvent.CreateStart(0, e)).
Concat(processNamesObs.Select(
e => StreamEvent.CreateEnd(10000, 0, e))).
ToStreamable();
pNamesStream2.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 58: Creating PNameStream using Interval and Edge Events");
var processNamesObs1 = new[]
{
new ProcessName(1, "Word"),
new ProcessName(2, "Internet Explorer"),
new ProcessName(3, "Excel"),
}.ToObservable();
var processNamesObs2 = new[]
{
new ProcessName(4, "Visual Studio"),
new ProcessName(5, "Outlook"),
}.ToObservable();
var pNamesStream3 = processNamesObs1.Select(e => StreamEvent.CreateInterval(0, 10000, e)).
Concat(processNamesObs2.Select(e => StreamEvent.CreateStart(0, e))).
Concat(processNamesObs2.Select(e => StreamEvent.CreateEnd(10000, 0, e))).
ToStreamable();
pNamesStream3.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 59: Coalescing Matching Edges");
pNamesStream3.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges).Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 60: Input with Punctuations");
var streamablePunctuations = new[]
{
StreamEvent.CreateInterval(0, 1, 1),
StreamEvent.CreateInterval(3, 4, 2),
StreamEvent.CreatePunctuation <int>(10),
StreamEvent.CreatePunctuation <int>(20),
StreamEvent.CreatePunctuation <int>(30),
StreamEvent.CreatePunctuation <int>(40),
StreamEvent.CreateInterval(40, 41, 3)
}.ToObservable().ToStreamable(DisorderPolicy.Drop());
streamablePunctuations.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
/*
* Console.WriteLine("Figure 61: Input with Generated Punctuations After Every Event");
* var streamableIthEventPunctuations = new[]
* {
* StreamEvent.CreateInterval(0, 1, 1),
* StreamEvent.CreateInterval(3, 4, 2),
* StreamEvent.CreateInterval(40, 41, 3)
* }.ToObservable().ToStreamable(DisorderPolicy.Drop(),
* PeriodicPunctuationPolicy.Count(1));
* streamableIthEventPunctuations.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
* Console.WriteLine();
*/
Console.WriteLine("Figure 62: Input with Generated Punctuations After 10 Tick Periods");
var streamableTimePeriodEventPunctuations = new[]
{
StreamEvent.CreateInterval(0, 1, 1),
StreamEvent.CreateInterval(10, 4, 2),
StreamEvent.CreateInterval(19, 4, 3),
StreamEvent.CreateInterval(40, 41, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Drop(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time(10));
streamableTimePeriodEventPunctuations.ToStreamEventObservable().Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 63: Query with No Output");
var IncompleteOutputQuery1 = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable().ToStreamable().Count().ToStreamEventObservable();
IncompleteOutputQuery1.Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 64: Query with Output up to Time 1");
var IncompleteOutputQuery2 = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2),
StreamEvent.CreatePunctuation <int>(1)
}.ToObservable().ToStreamable().Count().ToStreamEventObservable();
IncompleteOutputQuery2.Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 65: Alternate Query with Output up to Time 1");
var IncompleteOutputQuery3 = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable().ToStreamable().Count().ToStreamEventObservable();
IncompleteOutputQuery3.Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 66: Query with all Output");
var CompleteOutputQuery1 = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToObservable().ToStreamable().Count().ToStreamEventObservable();
CompleteOutputQuery1.Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 67: Alternate Query with all Output");
var CompleteOutputQuery2 = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable().ToStreamable().Count().
ToStreamEventObservable();
CompleteOutputQuery2.Where(e => e.IsData || e.IsPunctuation).ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine("The end ...");
#endregion
}
public static void Main(string[] args)
{
// This program is a supplement to the Trill Query Writing Guide
#region Section 2
IObservable <ContextSwitch> contextSwitchObservable = new[]
{
new ContextSwitch(0, 1, 1, 120),
new ContextSwitch(0, 3, 2, 121),
new ContextSwitch(0, 5, 3, 124),
new ContextSwitch(120, 2, 1, 123),
new ContextSwitch(300, 1, 1, 122),
new ContextSwitch(1800, 4, 2, 125),
new ContextSwitch(3540, 2, 1, 119),
new ContextSwitch(3600, 1, 1, 120),
}.ToObservable();
Console.WriteLine("Figure 11: Complete Pass-Through Trill Query Program");
IObservable <StreamEvent <ContextSwitch> > contextSwitchStreamEventObservable =
contextSwitchObservable.Select(e => StreamEvent.CreateInterval(e.Tick, e.Tick + 1, e));
IObservableIngressStreamable <ContextSwitch> contextSwitchIngressStreamable =
contextSwitchStreamEventObservable.ToStreamable(DisorderPolicy.Drop());
var contextSwitchStreamable =
(IStreamable <Empty, ContextSwitch>)contextSwitchIngressStreamable;
IObservable <StreamEvent <ContextSwitch> > passthroughContextSwitchStreamEventObservable =
contextSwitchStreamable.ToStreamEventObservable();
passthroughContextSwitchStreamEventObservable.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 12: Non-StreamEvent Passthrough");
var payloadStreamable = contextSwitchObservable.ToTemporalStreamable(cs => cs.Tick);
var passthroughPayloadObservable = payloadStreamable.ToTemporalObservable((start, cs) => cs);
passthroughPayloadObservable.ForEachAsync(cs => Console.WriteLine(cs.ToString())).Wait();
Console.WriteLine();
#endregion
#region Section 3
Console.WriteLine("Figure 14: Where Query Code");
var contextSwitchTwoCores = contextSwitchStreamable.Where(p => p.CpuId == 1 || p.CpuId == 2);
contextSwitchTwoCores.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 17: Select Query Code");
var contextSwitchTwoCoresNoTemp = contextSwitchTwoCores.Select(
e => new { e.Tick, e.ProcessId, e.CpuId });
contextSwitchTwoCoresNoTemp.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"Tick = {e.Payload.Tick, 4}\tProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 18: Alternate Select Query Code");
contextSwitchTwoCoresNoTemp = contextSwitchTwoCores.Select(
e => new { e.Tick, e.ProcessId, e.CpuId });
contextSwitchTwoCoresNoTemp.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"Tick = {e.Payload.Tick, 4}\tProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 20: Where and Select with the LINQ Comprehension Syntax");
contextSwitchTwoCoresNoTemp = from e in contextSwitchStreamable
where e.CpuId == 1 || e.CpuId == 2
select new { e.Tick, e.ProcessId, e.CpuId };
contextSwitchTwoCoresNoTemp.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"Tick = {e.Payload.Tick, 4}\tProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 4
Console.WriteLine("Figure 23: Join Query Code");
var processNamesObservable = new[]
{
new ProcessName(1, "Word"),
new ProcessName(2, "Internet Explorer"),
new ProcessName(3, "Excel"),
new ProcessName(4, "Visual Studio"),
new ProcessName(5, "Outlook"),
}.ToObservable();
var namesStream = processNamesObservable.
Select(e => StreamEvent.CreateInterval(0, 10000, e)).
ToStreamable();
var contextSwitchWithNames = contextSwitchTwoCoresNoTemp.Join(namesStream,
e => e.ProcessId, e => e.ProcessId,
(leftPayload, rightPayload) => new
{
leftPayload.Tick,
leftPayload.ProcessId,
leftPayload.CpuId,
rightPayload.Name
});
contextSwitchWithNames.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"Tick = {e.Payload.Tick, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 25: Join Query Comprehension Syntax");
contextSwitchWithNames = from leftPayload in contextSwitchTwoCoresNoTemp
join rightPayload in namesStream on
leftPayload.ProcessId equals rightPayload.ProcessId
select new
{
leftPayload.Tick,
leftPayload.ProcessId,
leftPayload.CpuId,
rightPayload.Name
};
contextSwitchWithNames.ToStreamEventObservable().Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"Tick = {e.Payload.Tick, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 26: Entire Join Query Comprehension Syntax");
contextSwitchWithNames = from leftPayload in contextSwitchStreamable
join rightPayload in namesStream on
leftPayload.ProcessId equals rightPayload.ProcessId
where leftPayload.CpuId == 1 || leftPayload.CpuId == 2
select new
{
leftPayload.Tick,
leftPayload.ProcessId,
leftPayload.CpuId,
rightPayload.Name
};
contextSwitchWithNames.ToStreamEventObservable().Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" + $"Tick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 5
Console.WriteLine("Figure 27: AlterEventDuration Query Code");
var infiniteContextSwitch = contextSwitchWithNames.AlterEventDuration(StreamEvent.InfinitySyncTime);
infiniteContextSwitch.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 20}\tTick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 6
Console.WriteLine("Figure 29: ClipEventDuration Query Code");
var clippedContextSwitch = infiniteContextSwitch
.ClipEventDuration(infiniteContextSwitch, e => e.CpuId, e => e.CpuId);
clippedContextSwitch.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 20}\tTick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 7
Console.WriteLine("Figure 31: Multicast Version of ClipEventDuration Query Code");
clippedContextSwitch = infiniteContextSwitch.Multicast(
s => s.ClipEventDuration(s, e => e.CpuId, e => e.CpuId));
clippedContextSwitch.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 20}\tTick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 8
Console.WriteLine("Figure 33: ShiftEventLifetime Query Code");
var shiftedClippedContextSwitch = clippedContextSwitch.ShiftEventLifetime(1);
shiftedClippedContextSwitch.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 20}\tTick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 35: Timeslices Query Code");
var timeslices = shiftedClippedContextSwitch.Join(contextSwitchWithNames,
e => e.CpuId, e => e.CpuId,
(left, right) => new
{
left.ProcessId,
left.CpuId,
left.Name,
Timeslice = right.Tick - left.Tick
});
timeslices.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}\tTimeslice = {e.Payload.Timeslice}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 37: Timeslices Query Code Using Multicast");
timeslices = contextSwitchWithNames.Multicast(t => t
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.Multicast(s => s
.ClipEventDuration(s, e => e.CpuId, e => e.CpuId))
.ShiftEventLifetime(1)
.Join(t,
e => e.CpuId, e => e.CpuId,
(left, right) => new
{
left.ProcessId,
left.CpuId,
left.Name,
Timeslice = right.Tick - left.Tick
}));
timeslices.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}\tTimeslice = {e.Payload.Timeslice}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 9
Console.WriteLine("Figure 41: Rassigning Timeslice Lifetimes with AlterEventLifetime");
var timeslicesForProcess1Cpu1 = timeslices.Where(e => e.ProcessId == 1 && e.CpuId == 1);
var windowedTimeslicesForProcess1Cpu1 = timeslicesForProcess1Cpu1.
AlterEventLifetime(origStartTime => (1 + ((origStartTime - 1) / 3600)) * 3600, 3600);
windowedTimeslicesForProcess1Cpu1.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name}\tTimeslice = {e.Payload.Timeslice}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 42: Reassigning Lifetimes with HoppingWindowLifetime");
var windowedTimeslices2 = timeslicesForProcess1Cpu1.HoppingWindowLifetime(3600, 3600);
windowedTimeslices2.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tProcessId = {e.Payload.ProcessId}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name}\tTimeslice = {e.Payload.Timeslice}"))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 43: Sum Query Code");
var totalConsumptionPerPeriodForProcess1Cpu1 = windowedTimeslicesForProcess1Cpu1.Sum(e => e.Timeslice);
totalConsumptionPerPeriodForProcess1Cpu1.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tPayload = {e.Payload}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 10
Console.WriteLine("Figure 45: Group and Apply Query Code");
var totalConsumptionPerPeriod = timeslices.GroupApply(
e => new { e.CpuId, e.ProcessId, e.Name },
s => s.HoppingWindowLifetime(3600, 3600).Sum(e => e.Timeslice),
(g, p) => new { g.Key.CpuId, g.Key.Name, TotalTime = p });
totalConsumptionPerPeriod.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\t" +
$"CpuId = {e.Payload.CpuId}\tName = {e.Payload.Name, 18}\tTotalTime = {e.Payload.TotalTime}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 11
Console.WriteLine("Figure 50: Chop Query Code");
Console.WriteLine("NB: This query is expected to fail!");
var alternativeContextSwitchStream = new[]
{
new ContextSwitch(0, 1, 1, 120),
new ContextSwitch(0, 3, 2, 121),
new ContextSwitch(0, 5, 3, 124),
new ContextSwitch(120, 2, 1, 123),
new ContextSwitch(300, 1, 1, 122),
new ContextSwitch(1800, 4, 2, 125),
new ContextSwitch(3540, 2, 1, 119),
new ContextSwitch(3600, 1, 1, 120),
new ContextSwitch(5400, 3, 2, 122),
new ContextSwitch(7200, 4, 2, 121),
}.ToObservable()
.Select(e => StreamEvent.CreateInterval(e.Tick, e.Tick + 1, e))
.ToStreamable(DisorderPolicy.Drop());
var contextSwitchChoppedUnbounded = alternativeContextSwitchStream
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.Multicast(s => s.ClipEventDuration(s, e => e.CpuId, e => e.CpuId))
.Chop(0, 3600)
.Select((origStartTime, e) =>
new { Tick = origStartTime, e.ProcessId, e.CpuId, e.CpuTemp })
.AlterEventDuration(1);
try
{
contextSwitchChoppedUnbounded.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tTick{e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tCPUTemp{e.Payload.CpuTemp}"))
.Wait();
}
catch (AggregateException e)
{
foreach (var ex in e.InnerExceptions)
{
Console.WriteLine("{0}", ex.Message);
}
}
catch (Exception e)
{
Console.WriteLine("{0}", e.Message);
}
Console.WriteLine();
Console.WriteLine("Figure 52: Improved Chop Query Code");
var fixedInterval = new[] { StreamEvent.CreateInterval(0, 10800, Unit.Default) }
.ToObservable().ToStreamable();
var contextSwitchChopped = alternativeContextSwitchStream
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.Multicast(s => s.ClipEventDuration(s, e => e.CpuId, e => e.CpuId))
.Join(fixedInterval, (left, right) => left)
.Chop(0, 3600)
.Select((origStartTime, e) =>
new { e.CpuId, e.ProcessId, e.CpuTemp, Tick = origStartTime })
.AlterEventDuration(1);
contextSwitchChopped.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"StartTime = {e.StartTime, 4}\tEndTime = {e.EndTime, 4}\tTick = {e.Payload.Tick, 4}\t" +
$"ProcessId = {e.Payload.ProcessId}\tCpuId = {e.Payload.CpuId}\tCPUTemp = {e.Payload.CpuTemp}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 12
Console.WriteLine("Figure 54: Final Query");
var choppedContextSwitch = alternativeContextSwitchStream
.Where(cs => cs.CpuId == 1 || cs.CpuId == 2)
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.Multicast(s => s.ClipEventDuration(s, e => e.CpuId, e => e.CpuId))
.Join(fixedInterval, (left, right) => left)
.Chop(0, 3600)
.Select((start, e) => new { Tick = start, e.ProcessId, e.CpuId, e.CpuTemp })
.AlterEventDuration(1);
var choppedContextSwitchWithNames = choppedContextSwitch
.Join(namesStream, e => e.ProcessId, e => e.ProcessId, (left, right) => new
{
left.Tick,
left.ProcessId,
left.CpuId,
right.Name
});
var timeslicesPerCpu = choppedContextSwitchWithNames
.Multicast(t => t
.AlterEventDuration(StreamEvent.InfinitySyncTime)
.Multicast(s => s.ClipEventDuration(s, e => e.CpuId, e => e.CpuId))
.ShiftEventLifetime(1)
.Join(t,
e => e.CpuId, e => e.CpuId,
(left, right) => new
{
left.ProcessId,
left.CpuId,
left.Name,
Timeslice = right.Tick - left.Tick
}));
var mostCpuConsumedPerPeriod = timeslicesPerCpu
.GroupApply(
e => new { e.ProcessId, e.Name },
s => s.HoppingWindowLifetime(3600, 3600).Sum(e => e.Timeslice),
(g, p) => new { g.Key.Name, TotalTime = p })
.Max((l, r) => l.TotalTime.CompareTo(r.TotalTime))
.Select((startTime, payload) => new
{
PeriodStart = startTime - 3600,
PeriodEnd = startTime,
ProcessName = payload.Name,
TotalCpuConsumed = payload.TotalTime
});
mostCpuConsumedPerPeriod.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine(
$"PeriodStart = {e.Payload.PeriodStart, 4}\tPeriodEnd = {e.Payload.PeriodEnd, 4}\t" +
$"Name = {e.Payload.ProcessName}\tTotalTime = {e.Payload.TotalCpuConsumed}"))
.Wait();
Console.WriteLine();
#endregion
#region Section 13.1
Console.WriteLine("Figure 56: Out of Order Input (Throw)");
var outOfOrderStreamableThrow = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Throw());
try
{
outOfOrderStreamableThrow.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
}
catch (Exception e)
{
Console.WriteLine("{0}", e.Message);
}
Console.WriteLine();
Console.WriteLine("Figure 57: Out of Order Input (Drop)");
var outOfOrderStreamableDrop = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Drop());
outOfOrderStreamableDrop.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 58: Out of Order Input (Adjust)");
var outOfOrderStreamableAdjust = new[]
{
StreamEvent.CreateInterval(10, 100, 1),
StreamEvent.CreateInterval(0, 50, 2),
StreamEvent.CreateInterval(0, 10, 3),
StreamEvent.CreateInterval(11, 90, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Adjust());
outOfOrderStreamableAdjust.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
#endregion
#region Section 13.3
Console.WriteLine("Figure 60: Creating namesStream with Edge Events");
namesStream = processNamesObservable.Select(e => StreamEvent.CreateStart(0, e)).
Concat(processNamesObservable.Select(e => StreamEvent.CreateEnd(10000, 0, e))).
ToStreamable();
namesStream.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 61: Creating namesStream using Interval and Edge Events");
var namesObservable1 = new[]
{
new ProcessName(1, "Word"),
new ProcessName(2, "Internet Explorer"),
new ProcessName(3, "Excel"),
}.ToObservable();
var namesObservable2 = new[]
{
new ProcessName(4, "Visual Studio"),
new ProcessName(5, "Outlook"),
}.ToObservable();
namesStream = namesObservable1.Select(e => StreamEvent.CreateInterval(0, 10000, e))
.Concat(namesObservable2.Select(e => StreamEvent.CreateStart(0, e)))
.Concat(namesObservable2.Select(e => StreamEvent.CreateEnd(10000, 0, e)))
.ToStreamable();
namesStream.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
Console.WriteLine("Figure 62: Coalescing Matching Edges");
namesStream.ToStreamEventObservable(ReshapingPolicy.CoalesceEndEdges)
.Where(e => e.IsData)
.ForEachAsync(e => WriteEvent(e))
.Wait();
Console.WriteLine();
#endregion
#region Section 13.4
Console.WriteLine("Figure 63: Input with Punctuations");
var streamablePunctuations = new[]
{
StreamEvent.CreateInterval(0, 1, 1),
StreamEvent.CreateInterval(3, 4, 2),
StreamEvent.CreatePunctuation <int>(10),
StreamEvent.CreatePunctuation <int>(20),
StreamEvent.CreatePunctuation <int>(30),
StreamEvent.CreatePunctuation <int>(40),
StreamEvent.CreateInterval(40, 41, 3)
}.ToObservable().ToStreamable(DisorderPolicy.Drop(), FlushPolicy.FlushOnPunctuation,
null, OnCompletedPolicy.None);
streamablePunctuations.ToStreamEventObservable()
.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 64: Input with Generated Punctuations After 10 Tick Periods");
var streamableTimePeriodEventPunctuations = new[]
{
StreamEvent.CreateInterval(0, 1, 1),
StreamEvent.CreateInterval(10, 4, 2),
StreamEvent.CreateInterval(19, 4, 3),
StreamEvent.CreateInterval(40, 41, 4)
}.ToObservable().ToStreamable(DisorderPolicy.Drop(), FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time(10), OnCompletedPolicy.None);
streamableTimePeriodEventPunctuations.ToStreamEventObservable()
.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 65: Query with No Output");
var incompleteOutputQuery = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable()
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, null, OnCompletedPolicy.None)
.Count()
.ToStreamEventObservable();
incompleteOutputQuery.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 66: Query with Output up to Time 1");
incompleteOutputQuery = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2),
StreamEvent.CreatePunctuation <int>(1)
}.ToObservable()
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, null, OnCompletedPolicy.None)
.Count()
.ToStreamEventObservable();
incompleteOutputQuery.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 67: Alternate Query with Output up to Time 1");
incompleteOutputQuery = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable()
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, null, OnCompletedPolicy.Flush)
.Count()
.ToStreamEventObservable();
incompleteOutputQuery.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 68: Query with all Output");
var completeOutputQuery = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToObservable()
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, null, OnCompletedPolicy.None)
.Count()
.ToStreamEventObservable();
completeOutputQuery.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
Console.WriteLine("Figure 69: Alternate Query with all Output");
completeOutputQuery = new[]
{
StreamEvent.CreateInterval(0, 10, 1),
StreamEvent.CreateInterval(1, 11, 2)
}.ToObservable()
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, null, OnCompletedPolicy.EndOfStream)
.Count()
.ToStreamEventObservable();
completeOutputQuery.ForEachAsync(e => WriteEvent(e)).Wait();
Console.WriteLine();
#endregion
Console.ReadLine();
}
public void LowWatermarkAndPunctuationGeneration()
{
SetupQuery(DisorderPolicy.Throw(), PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.Time(generationPeriod: 20),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 50, lowWatermarkTimestampLag: 0));
// This establishes three partitions with keys 0,1
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
// These points do not generate punctuations or low watermarks since neither period has lapsed.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 15, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 19, 1));
// These points generate punctuations (period of 20) but not low watermarks (period of 50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 22, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 20, 1));
// This point generates a low watermark, since it is >= the last watermark (none/0) + the low watermark
// generation period of 50
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 50, 0));
// These points do not generate punctuations, even though "punctuations" specifically haven't been generated
// for these partitions since time 22/20, because the low watermark at 50 supersedes punctuations.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 55, 1));
// Punctuation generation continues once the punctuation generation period lapses from the low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 70, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 71, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 91, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(0, 15, 0),
PartitionedStreamEvent.CreatePoint(1, 19, 1),
// Punctuations generated after the punctuation period of 20 from time 0, snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 20),
PartitionedStreamEvent.CreatePoint(0, 22, 0),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 20),
PartitionedStreamEvent.CreatePoint(1, 20, 1),
// Low watermark generated after the low watermark period of 50 from time 0
PartitionedStreamEvent.CreateLowWatermark <int, int>(50),
PartitionedStreamEvent.CreatePoint(0, 50, 0),
PartitionedStreamEvent.CreatePoint(1, 55, 1),
// Punctuations generated after the punctuation period of 20 from time 50 (low watermark time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 60),
PartitionedStreamEvent.CreatePoint(0, 70, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 60),
PartitionedStreamEvent.CreatePoint(1, 71, 1),
// Punctuations generated after the punctuation period of 20 from times 60 (last punctuation time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 80),
PartitionedStreamEvent.CreatePoint(0, 90, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 80),
PartitionedStreamEvent.CreatePoint(1, 91, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void LowWatermarkGenerationWithReorderLatency()
{
SetupQuery(
DisorderPolicy.Drop(reorderLatency: 50),
PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 100, lowWatermarkTimestampLag: 10));
// This establishes three partitions with keys 0,1,2
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// This will not yet generate a low watermark, since 99 - 0 is less than the generation period of 100
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 99, 0));
// This will generate a low watermark, since (110 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (in this case, none) + generation period (100).
// The timestamp will be 110 - (lowWatermarkTimestampLag of 10) = 100.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 110, 0));
// These will be dropped due to our PartitionedDisorderPolicy.Drop, since the low watermark is now 100,
// even as they approach the low watermark, since any event before the low watermark is out of order.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 99, 1));
// These will be honored, since they are >= low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 2));
// This will be reordered since it is >= low watermark and within the partition's
// high watermark (110) - reorderLatency (50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 120, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 150, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 190, 2));
// This will generate a low watermark, since (260 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (100) + generation period (100).
// The timestamp will be 260 - (lowWatermarkTimestampLag of 10) = 250.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 260, 0));
// An explicitly ingressed low watermark will update/reset the generationPeriod
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(300));
// Even though (370 - lowWatermarkTimestampLag of 10) is greater than the last automatically generated
// low watermark timestamp (100) + the generation period (100)); the lowWatermarkGeneration period was
// updated to start at 300, so the next generated low watermark will be at timestamp >=
// (300 + generationPeriod of 100)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 370, 1));
// This will generate a low watermark, since (410 - lowWatermarkTimestampLag of 10) is >= the last low
// watermark timestamp (300) + the generation period (100).
// The timestamp will be 410 - (lowWatermarkTimestampLag of 10) = 400.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 410, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
PartitionedStreamEvent.CreatePoint(0, 99, 0),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:100)
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
PartitionedStreamEvent.CreatePoint(0, 100, 1),
PartitionedStreamEvent.CreatePoint(2, 100, 0),
PartitionedStreamEvent.CreatePoint(1, 105, 2),
PartitionedStreamEvent.CreatePoint(2, 110, 0),
PartitionedStreamEvent.CreatePoint(0, 120, 0),
PartitionedStreamEvent.CreatePoint(1, 150, 1),
PartitionedStreamEvent.CreatePoint(2, 190, 2),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:260),
// snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreateLowWatermark <int, int>(200),
PartitionedStreamEvent.CreatePoint(2, 260, 0),
// Explicitly ingressed low watermark
PartitionedStreamEvent.CreateLowWatermark <int, int>(300),
PartitionedStreamEvent.CreatePoint(0, 370, 1),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 0:410)
PartitionedStreamEvent.CreateLowWatermark <int, int>(400),
PartitionedStreamEvent.CreatePoint(0, 410, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
