/// <summary>[Q4] “Find the most recent toll generated from vehicles being processed
/// at each station over a 3 minute window reporting the result every time a
/// change occurs in the input. Return the result at a point in time”.</summary>
private static void RunGroupedSlidingWindowDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count() // computed as a bonus, not asked in the query
};
BindAndRunQuery(
"GroupedSliding",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "TollAmount", "VehicleCount"
});
}
/// <summary>
/// Main entry point for the sample program.
/// </summary>
private static void Main()
{
using (Server server = Server.Create("Default"))
{
Application application = server.CreateApplication("Application");
// Define a simple query
CepStream <EventType> input = CepStream <EventType> .Create("Input");
var streamDefinition = from e in input
where e.X % 10 == 0
select e;
// Bind query
Uri inputAddress = new Uri("http://localhost:8080/InputAdapter");
Uri outputAddress = new Uri("http://localhost:8080/OutputAdapter");
Query query = CreateQuery(application, streamDefinition, inputAddress, outputAddress);
query.Start();
Console.WriteLine("Query started.");
Console.WriteLine("Press enter to stop query.");
// Produce and consume events via the service contract channels.
new Thread(() => ProduceEvents(inputAddress)).Start();
new Thread(() => ConsumeEvents(outputAddress)).Start();
Console.ReadLine();
query.Stop();
Console.WriteLine("Query stopped.");
}
}
/// <summary>[Q2] “Find the number of vehicles being processed in the toll
/// station at some time over a 3 minute window, with the time
/// advancing in one minute hops. Provide the counts as of the last
/// reported result. Return the result at a point in time at the
/// end of each 3 minute window” </summary>
private static void RunHoppingCountDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var tQ2 = from w in inputStream.HoppingWindow(
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(1),
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new TollCount
{
Count = w.Count()
};
var queryStream = from e in tQ2.ShiftEventTime(e => e.StartTime + TimeSpan.FromMinutes(3)).ToPointEventStream()
select e;
BindAndRunQuery(
"Hopping",
queryStream,
EventShape.Point,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Count"
});
}
/// <summary>[Q10] “For each vehicle that is being processed at an EZ-Pass booth, report
/// the TollReading if the tag does not exist, has expired, or is reported stolen</summary>
private static void RunUDFDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream
where (0 == e.Tag.Length) || IsLostOrStolen(e.Tag) || IsExpired(e.Tag)
select new TollViolation
{
LicensePlate = e.LicensePlate,
Make = e.Make,
Model = e.Model,
State = e.State,
Tag = e.Tag,
TollId = e.TollId
};
BindAndRunQuery(
"UDF",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"LicensePlate", "Make", "Model", "State", "Tag", "TollId"
});
}
/// <summary>
/// Contains the query logic in form of a query template.
/// </summary>
/// <param name="application">Application to host the query template.</param>
/// <returns>The new query template object.</returns>
private static QueryTemplate CreateQueryTemplate(Application application)
{
// Create stream objects as basis for query template
// specification. The specified names will be used when binding
// the query template's inputs to event producers.
CepStream <SensorReading> sensorStream = CepStream <SensorReading> .Create("sensorInput");
CepStream <LocationData> locationStream = CepStream <LocationData> .Create("locationInput");
// Extend duration of each sensor reading, so that they fall in
// a one-minute sliding window. Group by sensor ID and calculate the
// average vehicular count per group within each window.
// Include the grouping key in the aggregation result.
var avgCount = from oneMinReading in sensorStream.AlterEventDuration(e => TimeSpan.FromMinutes(1))
group oneMinReading by oneMinReading.SensorId into oneGroup
from eventWindow in oneGroup.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new { avgCount = eventWindow.Avg(e => e.VehicularCount), SensorId = oneGroup.Key };
// Join sensors and locations. Moreover, filter the count
// result by a threshold, which is looked up based on the
// sensor location through a user-defined function.
var joined = from averageEvent in avgCount
join locationData in locationStream
on averageEvent.SensorId equals locationData.SensorId
where averageEvent.avgCount > UserFunctions.LocationCountThreshold(locationData.LocationId)
select new
{
SensorId = locationData.SensorId,
LocationID = locationData.LocationId,
VehicularCount = averageEvent.avgCount
};
return(application.CreateQueryTemplate("SampleQueryTemplate", string.Empty, joined));
}
/// <summary>
/// Create a simple aggregate query.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreateAggregationQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Do some simple aggregation.
var q = from window in inputStream.HoppingWindow(TimeSpan.FromHours(1), TimeSpan.FromMinutes(1))
select new { AvgX = window.Avg(e => e.X), AvgY = window.Avg(e => e.Y) };
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName)), new string[] { "AvgX", "AvgY" }),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>[Q11] “Over a 3 minute tumbling window, find the ratio of out-of-state
/// vehicles to total vehicles being processed at a toll station”</summary>
private static void RunUDADemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(
TimeSpan.FromMinutes((double)3),
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new VehicleRatio
{
Value = w.OutOfStateVehicleRatio()
};
BindAndRunQuery(
"UDA",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Value"
});
}
private static void RunQuery(Application cepApplication)
{
var config = new YahooDataInputConfig()
{
Symbols = new string[] { "AAPL", "DELL", "MSFT", "GOOG", "GE" },
RefreshInterval = TimeSpan.FromSeconds(0.5),
TimestampIncrement = TimeSpan.FromSeconds(0.5),
AlwaysUseNow = true,
EnqueueCtis = false
};
AdvanceTimeSettings ats = new AdvanceTimeSettings(new AdvanceTimeGenerationSettings(
TimeSpan.FromMilliseconds(1000),
TimeSpan.FromMilliseconds(200)),
null, AdvanceTimePolicy.Drop);
var data = CepStream <YahooDataEvent> .Create(cepApplication, "TestData", typeof(YahooDataInputFactory), config,
EventShape.Point, ats);
var query = data.ToQuery(cepApplication, "Test", "Test", typeof(ConsoleOutputFactory),
new ConsoleOutputConfig()
{
ShowCti = true,
CtiEventColor = ConsoleColor.Yellow,
InsertEventColor = ConsoleColor.Magenta
},
EventShape.Point, StreamEventOrder.FullyOrdered);
query.Start();
}
/// <summary>
/// Create a simple passthrough query.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreatePassthroughQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Just a passthrough template.
var q = from e in inputStream select e;
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName))),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>[Q3] “Find the toll generated from vehicles being processed at each
/// toll station at some time over a 3 minute window, with the time advancing
/// in one minute hops. Provide the value as of the last reported result.
/// Return the result at a point in time” </summary>
private static void RunGroupedHoppingWindow()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream
group e by e.TollId into perTollBooth
from w in perTollBooth.HoppingWindow(
TimeSpan.FromMinutes(3), // Window Size
TimeSpan.FromMinutes(1), // Hop Size
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
BindAndRunQuery(
"GroupedHopping",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "TollAmount", "VehicleCount"
});
}
static void Main(string[] args)
{
var server = Server.Create("default");
var application = server.CreateApplication("CellData");
// Configuration for input 1
var input1Config = new SensorInputConfig
{
Timeout = Program.Timeout,
Interval = Program.Interval,
NumberOfReadings = Program.NumberOfReading
};
//string streamName = "{ \"sensor\":\"compass\",\"values\":{ \"x\":82.26953,\"y\":-17.796535,\"z\":-8.009412},\"type\":\"sensor\"}";
// Instantiate input adapters
var input1Stream = CepStream <SensorData> .Create("input1Stream", typeof(SensorInputAdapterFactory), input1Config, EventShape.Point);
// Configure output adapter
var outputConfig = new SensorOutputConfig {
AdapterStopSignal = "StopData"
};
// filtering logic for the json data
// Join input adapters with a simple LINQ query
var combinedInputStream = (from e in input1Stream
//where (e.sensor == "compass" && Convert.ToDouble(e.values.x) > 149)
//select e);
where (e.sensor == "compass")
select new SensorData
{
sensor = e.sensor,
values = e.values,
type = e.type,
direction = Direction.GetDirection(e.values.x)
});
// Connect input adapters with output adapter
var query = combinedInputStream.ToQuery(application, "CellInformation", "...", typeof(SensorOutputAdapterFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Instantiate semaphor for stop signal
var adapterStopSignal = new EventWaitHandle(false,
EventResetMode.ManualReset, outputConfig.AdapterStopSignal);
// Run the query
query.Start();
adapterStopSignal.WaitOne();
query.Stop();
application.Delete();
server.Dispose();
}
public void TestTwitter()
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("twittertest");
var inputConfig = new TwitterConfig
{
Username = UserNameTbx.Text,
Password = PasswordTbx.Text
};
if (FilterModeRbn.Checked)
{
inputConfig.Mode = TwitterMode.Filter;
inputConfig.Parameters = "track=" + FilterParametersTbx.Text;
}
else if (SampleModeRbn.Checked)
{
inputConfig.Mode = TwitterMode.Sample;
}
var input = CepStream <Tweet> .Create("twitter", typeof(TwitterFactory), inputConfig, EventShape.Point);
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
// Create query and bind to the output adapter
query = input.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
application.Delete();
server.Dispose();
}
public static void Main()
{
var cepStream = CepStream <MediationData> .Create("cepStream1");
Console.WriteLine("\n-----------------------------Filter e.Number > 3\n");
var filter = from e in cepStream
where e.Number > 3
select e;
RunCepStream(filter);
Console.WriteLine("\n-----------------------------Get average for HoppingWindow(3,1)\n");
var avgCepStream = from w in cepStream.Where(e => true).HoppingWindow(TimeSpan.FromMinutes(3), TimeSpan.FromMinutes(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new MediationData()
{
Direction = "Average",
Number = w.Avg(e => e.Number)
};
RunCepStream(avgCepStream);
Console.WriteLine("\n-----------------------------Sum of TumblingWindow(3)\n");
var count = from win in cepStream.TumblingWindow(TimeSpan.FromMinutes(3))
select new MediationData()
{
Direction = "Average",
Number = win.Sum(e => e.Number)
};
RunCepStream(count);
Console.WriteLine("\n-----------------------------Group e.Direction With HoppingWindow(3,3)\n");
var groupCepStream = from e in cepStream
group e by e.Direction into eGroup
from w in eGroup.HoppingWindow(TimeSpan.FromMinutes(3), TimeSpan.FromMinutes(3), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new MediationData()
{
Direction = eGroup.Key,
Number = w.Avg(e => e.Number)
};
RunCepStream(groupCepStream);
Console.WriteLine("\n-----------------------------ShiftEvent by 7 min\n");
var shiftEventCepStream = from e in cepStream.ShiftEventTime(e => e.StartTime.AddMinutes(7))
select e;
RunCepStream(shiftEventCepStream);
application.Delete();
server.Dispose();
}
public void TestFacebook()
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("facebooktest");
var inputConfig = new FacebookConfig
{
AccessToken = AccessTokenTbx.Text,
UsernameOrUniqueId = UsernameOrUniqueIdTbx.Text,
RefreshPeriod = int.Parse(RefreshIntervalTbx.Text) * 1000
};
var input = CepStream <FacebookItem> .Create("facebook", typeof(FacebookFactory), inputConfig, EventShape.Point);
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
// Create query and bind to the output adapter
var expr = from e in input
select new { e.Id, e.FromName, Message = !string.IsNullOrEmpty(e.Message) ? e.Message : e.Description };
query = expr.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.FullyOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
application.Delete();
server.Dispose();
}
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("yahoofinancetest");
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
var inputConfig = new YahooFinanceConfig
{
StockSymbol = "^GSPC",
StockName = "S&P 500",
Timeout = 10000,
Interval = 1000
};
var input = CepStream <StockQuote> .Create("yahoo", typeof(YahooFinanceFactory), inputConfig, EventShape.Point);
// Create query and bind to the output adapter
query = input.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
Thread.Sleep(1000);
application.Delete();
server.Dispose();
}
/// <summary>
/// Example of using a user-defined filter
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createUserFilterExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var filteredCepStream = from e in input
where UserDefinedFilter(e.FieldID)
select e;
var queryTemplate = application.CreateQueryTemplate("userFilterExampleTemplate", "Description...", filteredCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("userFilterExampleQuery", "Description...", queryBinder);
return(query);
}
/// <summary>[Q5] “Report the output whenever Toll Booth 2 has the same number
/// of vehicles as Toll Booth 1, every time a change occurs in either stream”</summary>
private static void RunJoinDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var slidingWindowQuery = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
var stream1 = from e in slidingWindowQuery
where e.TollId.Equals(((int)1).ToString())
select e;
var stream2 = from e in slidingWindowQuery
where e.TollId.Equals(((int)2).ToString())
select e;
var queryStream = from e1 in stream1
join e2 in stream2
on e1.VehicleCount equals e2.VehicleCount
select new TollCompare
{
TollId1 = e1.TollId,
TollId2 = e2.TollId,
VehicleCount = e1.VehicleCount
};
BindAndRunQuery(
"Join",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId1", "TollId2", "VehicleCount"
});
}
/// <summary>[Q12] “Over a one hour tumbling window, report all commercial vehicles
/// with tonnage greater than one ton (2K lbs), along with their arrival
/// times at the toll, and any charges due to weight violation. Overweight
/// charges during the rush hour (7am to 7pm) are double that of non-rush hours”</summary>
private static void RunUDODemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(TimeSpan.FromHours((double)1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select w.VehicleWeights();
BindAndRunQuery(
"UDO",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"LicensePlate", "Weight", "WeightCharge"
});
}
private static void RunFilterProject()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream
where w.TollId == "1"
select w;
BindAndRunQuery(
"Tumbling",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
});
}
/// <summary>
/// Bucket things based on position and take counts.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreateBucketQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Do some aggregation by leading digit of x and y coordinate.
var q = from e in inputStream
group e by new { XBucket = (int)(e.X * 10), YBucket = (int)(e.Y * 10) } into xygroups
from window in xygroups.TumblingWindow(TimeSpan.FromDays(1))
select new
{
XBucket = xygroups.Key.XBucket,
YBucket = xygroups.Key.YBucket,
Count = window.Count(),
AvgX = window.Avg(e => e.X),
AvgY = window.Avg(e => e.Y)
};
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName)), new string[] { "XBucket", "YBucket", "AvgX", "AvgY", "Count" }),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>
/// Compare USD and SEK stock prices using given exchange rate. Example of a cross join.
/// </summary>
/// <param name="application"></param>
/// <param name="ericUSDConfig"></param>
/// <param name="ericSEKConfig"></param>
/// <param name="USDSEKConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createCrossJoinExampleQuery(Application application, StockQuoteInputConfig ericUSDConfig, StockQuoteInputConfig ericSEKConfig, StockQuoteInputConfig USDSEKConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var ericUSDStream = CepStream <StockQuote> .Create("ericUSDStream");
var ericSEKStream = CepStream <StockQuote> .Create("ericSEKStream");
var USDSEKStream = CepStream <StockQuote> .Create("USDSEKStream");
var ericRecalcCepStream = from eUSD in ericUSDStream
from eXch in USDSEKStream // Cross join
where eUSD.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Recalc",
FieldID = "Close",
Value = eUSD.Value * eXch.Value // Convert ERIC USD quote to SEK
};
var ericCompareCepStream = from eRecalc in ericRecalcCepStream
from eSEK in ericSEKStream
where eSEK.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Compare",
FieldID = "Diff",
Value = eSEK.Value - eRecalc.Value
};
var queryTemplate = application.CreateQueryTemplate("ericCompareTemplate", "Description...", ericCompareCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("ericUSDStream", inputAdapter, ericUSDConfig, EventShape.Point);
queryBinder.BindProducer <StockQuote>("ericSEKStream", inputAdapter, ericSEKConfig, EventShape.Point);
queryBinder.BindProducer <StockQuote>("USDSEKStream", inputAdapter, USDSEKConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("ericCompareQuery", "Description...", queryBinder);
return(query);
}
/// <summary>[Q8] “Report the top 2 toll amounts from the results of Q4 over a 3 minute tumbling window”</summary>
private static void RunTopKDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var groupedSliding = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
var queryStream = (from window in groupedSliding.TumblingWindow(TimeSpan.FromMinutes((double)3), HoppingWindowOutputPolicy.ClipToWindowEnd)
from e in window
orderby e.TollAmount descending
select e).Take(
(uint)2,
e => new TopEvents
{
TollRank = e.Rank,
TollAmount = e.Payload.TollAmount,
TollId = e.Payload.TollId,
VehicleCount = e.Payload.VehicleCount
});
BindAndRunQuery(
"Top-K",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollRank", "TollAmount", "TollId", "VehicleCount"
});
}
/// <summary>[tQ1] "Every 3 minutes, report the number of vehicles processed
/// that were being processed at some point during that period at
/// the toll station since the last result. Report the result at a
/// point in time, at the beginning of the 3 minute window"</summary>
private static void RunTumblingCountDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(TimeSpan.FromMinutes(3), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new TollCount {
Count = w.Count()
};
BindAndRunQuery(
"Tumbling",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Count"
});
}
/// <summary>
/// Example of using a user-defined aggregate
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createStandardDeviationExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var stddevCepStream = from w in input.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC",
FieldID = "7-day Stddev",
Value = w.StandardDeviation()
};
var queryTemplate = application.CreateQueryTemplate("standardDeviationExampleTemplate", "Description...", stddevCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("standardDeviationExampleQuery", "Description...", queryBinder);
return(query);
}
/// <summary>
/// Example of a grouping and calculation of averages for the groups
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createGroupApplyExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var ericUSDGroupCepStream = from e in input
group e by e.FieldID into eGroup
from w in eGroup.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC 7-day avg",
FieldID = eGroup.Key,
Value = w.Avg(e => e.Value)
};
var queryTemplate = application.CreateQueryTemplate("groupApplyExampleTemplate", "Description...", ericUSDGroupCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("groupApplyExampleQuery", "Description...", queryBinder);
return(query);
}
/// <summary>
/// Example of detecting when stock price falls more than 10% in 7 days
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createBigLooserExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var bigLooserCepStream = (from e1 in input
from e2 in input.ShiftEventTime(e => e.StartTime.AddDays(7))
where e1.FieldID == "Close" && e2.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC > 10% drop",
FieldID = "Close",
Value = (e1.Value - e2.Value) / e2.Value * 100
}).Where(e => e.Value < -10);
var queryTemplate = application.CreateQueryTemplate("bigLooserExampleTemplate", "Description...", bigLooserCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("bigLooserExampleQuery", "Description...", queryBinder);
return(query);
}
/// <summary>[Q7] "Report the output whenever a vehicle passes a EZ-Pass booth with no
/// tags (i.e. report a toll violation)”</summary>
private static void RunLASJDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
// Simulate the reference stream from inputStream itself - convert it to a point event stream
var referenceStream = from e in inputStream.ToPointEventStream() select e;
// Simulate the tag violations in the observed stream by filtering out specific
// vehicles. Let us filter out all the events in the dataset TollInput.txt with
// a Tag length of 0 (this is the last element in each line in TollInput.txt).
// In a real scenario, these events will not exist at all – this simulation is only
// because we are reusing the same input stream for this example.
// The events that were filtered out should be the ones that show up in the output of tQ7
var observedStream = from e in inputStream.ToPointEventStream()
where 0 != e.Tag.Length
select e;
// Report tag violations
var queryStream = from left in referenceStream
where (from right in observedStream
select right).IsEmpty()
select left;
BindAndRunQuery(
"LASJ",
queryStream,
EventShape.Point,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
});
}
internal static void Main()
{
using (var server = Server.Create("Default"))
{
// If you want to publish this server, or connect to an already existing server,
// please see the product documentation "Publishing and Connecting to a StreamInsight Server"
// to find out how to replace, or add to the above server creation code. You must publish
// the server for the Event Flow debugger to be able to connect to the server.
var application = server.CreateApplication("TutorialApp");
// Define device configuration
var inputConfig = new CsvInputConfig
{
InputFileName = @"..\..\..\TollInput.txt",
Delimiter = new char[] { ',' },
BufferSize = 4096,
CtiFrequency = 1,
CultureName = "en-US",
Fields = new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
NonPayloadFieldCount = 2,
StartTimePos = 1,
EndTimePos = 2
};
var outputConfigForPassThroughQuery = new CsvOutputConfig
{
// The adapter recognizes empty filename as a write to console
OutputFileName = string.Empty,
Delimiter = new string[] { "\t" },
CultureName = "en-US",
Fields = new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
}
};
var outputConfigForCountQuery = new CsvOutputConfig
{
OutputFileName = string.Empty,
Delimiter = new string[] { "\t" },
CultureName = "en-US",
Fields = new List <string>()
{
"Count"
}
};
// Define console trace listener
TraceListener tracer = new ConsoleTraceListener();
// set up advance time settings to enqueue CTIs - to deliver output in a timely fashion
var advanceTimeGenerationSettings = new AdvanceTimeGenerationSettings(inputConfig.CtiFrequency, TimeSpan.Zero, true);
var advanceTimeSettings = new AdvanceTimeSettings(advanceTimeGenerationSettings, null, AdvanceTimePolicy.Adjust);
// Define input stream object, mapped to stream names
// Instantiate an adapter from the input factory class
var inputStream = CepStream <TollReading> .Create(
"TollStream",
typeof(CsvInputFactory),
inputConfig,
EventShape.Interval,
advanceTimeSettings);
var passthroughQuery = from e in inputStream select e;
var countQuery = from w in inputStream.TumblingWindow(TimeSpan.FromMinutes(3), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new TollCount {
Count = w.Count()
};
// Create a runnable query by binding the query template to the
// input stream of interval events, and to an output stream of
// fully ordered point events (through an output adapter instantiated
// from the output factory class)
Query query = countQuery.ToQuery(
application,
"HelloTollTutorial",
"Hello Toll Query",
typeof(CsvOutputFactory),
outputConfigForCountQuery,
EventShape.Interval,
StreamEventOrder.FullyOrdered);
query.Start();
Console.WriteLine("*** Hit Return to see Query Diagnostics after this run ***");
Console.WriteLine();
Console.ReadLine();
// Retrieve diagnostic information from the CEP server about the query.
// See the section "Connecting to and Publishing a Server" in the StreamInsight MSDN documentation on
// how to make the debugger client connect to the server and retrieve these diagnostics
Console.WriteLine(string.Empty);
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/EventManager")), tracer);
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/PlanManager")), tracer);
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/Application/TutorialApp/Query/HelloTollTutorial")), tracer);
DiagnosticSettings settings = new DiagnosticSettings(DiagnosticAspect.GenerateErrorReports, DiagnosticLevel.Always);
server.SetDiagnosticSettings(new Uri("cep:/Server"), settings);
tracer.WriteLine("Global Server Diagnostics");
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/EventManager")), tracer);
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/PlanManager")), tracer);
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/Query")), tracer);
tracer.WriteLine(string.Empty);
tracer.WriteLine("Summary Query Diagnostics for");
RetrieveDiagnostics(server.GetDiagnosticView(new Uri("cep:/Server/Application/TutorialApp/Query/HelloTollTutorial")), tracer);
tracer.WriteLine(string.Empty);
query.Stop();
Console.WriteLine("*** Query Completed *** Hit Return to Exit");
Console.WriteLine();
Console.ReadLine();
}
return;
}
public static void Main(string [] args)
{
// Create a tracer to output information on the console.
TraceListener tracer = new ConsoleTraceListener();
using (Server server = Server.Create("MyInstance"))
{
try
{
Application application = server.CreateApplication("application");
var ericSEKConfig = new InputConfig
{
ID = "ERIC-SEK",
Filename = "eric_b_sek_2009.csv",
ColumnNames = new string[] { "Open", "High", "Low", "Close", "Volume", "Adj Close" },
StartDate = new DateTime(2009, 01, 01),
Interval = 0
};
// Implicit 方式
var cepStream = CepStream <Payload> .Create("cepStream",
typeof(PayloadInputFactory),
ericSEKConfig,
EventShape.Point);
var filtered = from e in cepStream
where e.Value > 95
select e;
/*
* var query = filtered.ToQuery(application,
* "filterQuery",
* "Filter out Values over 95",
* typeof(PayloadOutputFactory),
* new PayloadOutputConfig(),
* EventShape.Point,
* StreamEventOrder.FullyOrdered);
*/
RunQuery(filtered, application);
var avgCepStream = from w in cepStream.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new Payload()
{
StockID = "ERIC",
FieldID = "7-day avg",
Value = w.Avg(e => e.Value)
};
RunQuery(avgCepStream, application);
var ericUSDGroupCepStream = from e in cepStream
group e by e.FieldID into eGroup
from w in eGroup.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new Payload()
{
StockID = "ERIC 7-day avg",
FieldID = eGroup.Key,
Value = w.Avg(e => e.Value)
};
RunQuery(ericUSDGroupCepStream, application);
var bigLooserCepStream = (from e1 in cepStream
from e2 in cepStream.ShiftEventTime(e => e.StartTime.AddDays(7))
where e1.FieldID == "Close" && e2.FieldID == "Close"
select new Payload()
{
StockID = "ERIC > 10% drop",
FieldID = "Close",
Value = (e1.Value - e2.Value) / e2.Value * 100
}).Where(e => e.Value < -10);
RunQuery(bigLooserCepStream, application);
// Explicit 方式
var input = CepStream <Payload> .Create("input");
var stddevCepStream = from w in input.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new Payload()
{
StockID = "ERIC",
FieldID = "7-day Stddev",
Value = w.StandardDeviation()
};
var queryTemplate = application.CreateQueryTemplate("standardDeviationExampleTemplate", "Description...", stddevCepStream);
var queryBinder = new QueryBinder(queryTemplate);
var InputAdapterFromFile = application.CreateInputAdapter <PayloadInputFactory>("PayloadInput", "Description...");
var outputAdapter = application.CreateOutputAdapter <PayloadOutputFactory>("PayloadOutput", "Description...");
queryBinder.BindProducer <Payload>("input", InputAdapterFromFile, ericSEKConfig, EventShape.Point);
queryBinder.AddConsumer <Payload>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("standardDeviationExampleQuery", "Description...", queryBinder);
query.Start();
// Wait until output adapter signals that it is finished
DiagnosticView diagnosticView;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while ((string)diagnosticView["QueryState"] == "Running");
// Stop
query.Stop();
// Release resources
application.Delete();
server.Dispose();
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
}
/// <summary>[QouterJoin] - Outer Join using query primitives</summary>
private static void RunOuterJoinDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
// Simulate the left stream input from inputStream
var outerJoin_L = from e in inputStream
select new
{
LicensePlate = e.LicensePlate,
Make = e.Make,
Model = e.Model,
};
// Simulate the right stream input from inputStream – eliminate all events with Toyota as the vehicle
// These should be the rows in the outer joined result with NULL values for Toll and LicensePlate
var outerJoin_R = from e in inputStream
where !e.Make.Equals("Toyota")
select new
{
LicensePlate = e.LicensePlate,
Toll = e.Toll,
TollId = e.TollId
};
// Inner join the two simulated input streams
var innerJoin = from left in outerJoin_L
from right in outerJoin_R
where left.LicensePlate.Equals(right.LicensePlate)
select new TollOuterJoin
{
LicensePlate = left.LicensePlate,
Make = left.Make,
Model = left.Model,
Toll = right.Toll,
TollId = right.TollId
};
// Left anti-semijoin the two input simulated streams, and add the Project
var outerJoin_lasj = from left in outerJoin_L
where (from right in outerJoin_R
where left.LicensePlate == right.LicensePlate
select right).IsEmpty()
select new TollOuterJoin
{
LicensePlate = left.LicensePlate,
Make = left.Make,
Model = left.Model,
Toll = null,
TollId = null
};
// Union the two streams to complete a Left Outer Join operation
var queryStream = innerJoin.Union(outerJoin_lasj);
BindAndRunQuery(
"OuterJoin",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"LicensePlate", "Make", "Model", "Toll", "TollId"
});
}