public static void RunCepStream(CepStream <MediationData> cepStream)
{
var binder = new QueryBinder(application.CreateQueryTemplate(
Guid.NewGuid().ToString(),
"description1",
cepStream));
binder.BindProducer <MediationData>(
"cepStream1",
inputAdapter,
inConfig,
EventShape.Point);
binder.AddConsumer <MediationData>(
"output1",
outputAdapter,
outConfig,
EventShape.Point,
StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("queyName", "desc", binder);
query.Start();
DiagnosticView diagnosticView;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while ((string)diagnosticView["QueryState"] == "Running");
query.Stop();
query.Delete();
}
/// <summary>[Q4] “Find the most recent toll generated from vehicles being processed
/// at each station over a 3 minute window reporting the result every time a
/// change occurs in the input. Return the result at a point in time”.</summary>
private static void RunGroupedSlidingWindowDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count() // computed as a bonus, not asked in the query
};
BindAndRunQuery(
"GroupedSliding",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "TollAmount", "VehicleCount"
});
}
private static void RunQuery(Application cepApplication)
{
var config = new YahooDataInputConfig()
{
Symbols = new string[] { "AAPL", "DELL", "MSFT", "GOOG", "GE" },
RefreshInterval = TimeSpan.FromSeconds(0.5),
TimestampIncrement = TimeSpan.FromSeconds(0.5),
AlwaysUseNow = true,
EnqueueCtis = false
};
AdvanceTimeSettings ats = new AdvanceTimeSettings(new AdvanceTimeGenerationSettings(
TimeSpan.FromMilliseconds(1000),
TimeSpan.FromMilliseconds(200)),
null, AdvanceTimePolicy.Drop);
var data = CepStream <YahooDataEvent> .Create(cepApplication, "TestData", typeof(YahooDataInputFactory), config,
EventShape.Point, ats);
var query = data.ToQuery(cepApplication, "Test", "Test", typeof(ConsoleOutputFactory),
new ConsoleOutputConfig()
{
ShowCti = true,
CtiEventColor = ConsoleColor.Yellow,
InsertEventColor = ConsoleColor.Magenta
},
EventShape.Point, StreamEventOrder.FullyOrdered);
query.Start();
}
/// <summary>
/// Create a simple passthrough query.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreatePassthroughQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Just a passthrough template.
var q = from e in inputStream select e;
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName))),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>[Q2] “Find the number of vehicles being processed in the toll
/// station at some time over a 3 minute window, with the time
/// advancing in one minute hops. Provide the counts as of the last
/// reported result. Return the result at a point in time at the
/// end of each 3 minute window” </summary>
private static void RunHoppingCountDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var tQ2 = from w in inputStream.HoppingWindow(
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(1),
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new TollCount
{
Count = w.Count()
};
var queryStream = from e in tQ2.ShiftEventTime(e => e.StartTime + TimeSpan.FromMinutes(3)).ToPointEventStream()
select e;
BindAndRunQuery(
"Hopping",
queryStream,
EventShape.Point,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Count"
});
}
/// <summary>[Q11] “Over a 3 minute tumbling window, find the ratio of out-of-state
/// vehicles to total vehicles being processed at a toll station”</summary>
private static void RunUDADemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(
TimeSpan.FromMinutes((double)3),
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new VehicleRatio
{
Value = w.OutOfStateVehicleRatio()
};
BindAndRunQuery(
"UDA",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Value"
});
}
/// <summary>
/// Contains the query logic in form of a query template.
/// </summary>
/// <param name="application">Application to host the query template.</param>
/// <returns>The new query template object.</returns>
private static QueryTemplate CreateQueryTemplate(Application application)
{
// Create stream objects as basis for query template
// specification. The specified names will be used when binding
// the query template's inputs to event producers.
CepStream <SensorReading> sensorStream = CepStream <SensorReading> .Create("sensorInput");
CepStream <LocationData> locationStream = CepStream <LocationData> .Create("locationInput");
// Extend duration of each sensor reading, so that they fall in
// a one-minute sliding window. Group by sensor ID and calculate the
// average vehicular count per group within each window.
// Include the grouping key in the aggregation result.
var avgCount = from oneMinReading in sensorStream.AlterEventDuration(e => TimeSpan.FromMinutes(1))
group oneMinReading by oneMinReading.SensorId into oneGroup
from eventWindow in oneGroup.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new { avgCount = eventWindow.Avg(e => e.VehicularCount), SensorId = oneGroup.Key };
// Join sensors and locations. Moreover, filter the count
// result by a threshold, which is looked up based on the
// sensor location through a user-defined function.
var joined = from averageEvent in avgCount
join locationData in locationStream
on averageEvent.SensorId equals locationData.SensorId
where averageEvent.avgCount > UserFunctions.LocationCountThreshold(locationData.LocationId)
select new
{
SensorId = locationData.SensorId,
LocationID = locationData.LocationId,
VehicularCount = averageEvent.avgCount
};
return(application.CreateQueryTemplate("SampleQueryTemplate", string.Empty, joined));
}
/// <summary>
/// Main entry point for the sample program.
/// </summary>
private static void Main()
{
using (Server server = Server.Create("Default"))
{
Application application = server.CreateApplication("Application");
// Define a simple query
CepStream <EventType> input = CepStream <EventType> .Create("Input");
var streamDefinition = from e in input
where e.X % 10 == 0
select e;
// Bind query
Uri inputAddress = new Uri("http://localhost:8080/InputAdapter");
Uri outputAddress = new Uri("http://localhost:8080/OutputAdapter");
Query query = CreateQuery(application, streamDefinition, inputAddress, outputAddress);
query.Start();
Console.WriteLine("Query started.");
Console.WriteLine("Press enter to stop query.");
// Produce and consume events via the service contract channels.
new Thread(() => ProduceEvents(inputAddress)).Start();
new Thread(() => ConsumeEvents(outputAddress)).Start();
Console.ReadLine();
query.Stop();
Console.WriteLine("Query stopped.");
}
}
/// <summary>[Q3] “Find the toll generated from vehicles being processed at each
/// toll station at some time over a 3 minute window, with the time advancing
/// in one minute hops. Provide the value as of the last reported result.
/// Return the result at a point in time” </summary>
private static void RunGroupedHoppingWindow()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream
group e by e.TollId into perTollBooth
from w in perTollBooth.HoppingWindow(
TimeSpan.FromMinutes(3), // Window Size
TimeSpan.FromMinutes(1), // Hop Size
HoppingWindowOutputPolicy.ClipToWindowEnd)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
BindAndRunQuery(
"GroupedHopping",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "TollAmount", "VehicleCount"
});
}
/// <summary>
/// Create a simple aggregate query.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreateAggregationQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Do some simple aggregation.
var q = from window in inputStream.HoppingWindow(TimeSpan.FromHours(1), TimeSpan.FromMinutes(1))
select new { AvgX = window.Avg(e => e.X), AvgY = window.Avg(e => e.Y) };
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName)), new string[] { "AvgX", "AvgY" }),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>[Q10] “For each vehicle that is being processed at an EZ-Pass booth, report
/// the TollReading if the tag does not exist, has expired, or is reported stolen</summary>
private static void RunUDFDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from e in inputStream
where (0 == e.Tag.Length) || IsLostOrStolen(e.Tag) || IsExpired(e.Tag)
select new TollViolation
{
LicensePlate = e.LicensePlate,
Make = e.Make,
Model = e.Model,
State = e.State,
Tag = e.Tag,
TollId = e.TollId
};
BindAndRunQuery(
"UDF",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"LicensePlate", "Make", "Model", "State", "Tag", "TollId"
});
}
static void Main(string[] args)
{
var server = Server.Create("default");
var application = server.CreateApplication("CellData");
// Configuration for input 1
var input1Config = new SensorInputConfig
{
Timeout = Program.Timeout,
Interval = Program.Interval,
NumberOfReadings = Program.NumberOfReading
};
//string streamName = "{ \"sensor\":\"compass\",\"values\":{ \"x\":82.26953,\"y\":-17.796535,\"z\":-8.009412},\"type\":\"sensor\"}";
// Instantiate input adapters
var input1Stream = CepStream <SensorData> .Create("input1Stream", typeof(SensorInputAdapterFactory), input1Config, EventShape.Point);
// Configure output adapter
var outputConfig = new SensorOutputConfig {
AdapterStopSignal = "StopData"
};
// filtering logic for the json data
// Join input adapters with a simple LINQ query
var combinedInputStream = (from e in input1Stream
//where (e.sensor == "compass" && Convert.ToDouble(e.values.x) > 149)
//select e);
where (e.sensor == "compass")
select new SensorData
{
sensor = e.sensor,
values = e.values,
type = e.type,
direction = Direction.GetDirection(e.values.x)
});
// Connect input adapters with output adapter
var query = combinedInputStream.ToQuery(application, "CellInformation", "...", typeof(SensorOutputAdapterFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Instantiate semaphor for stop signal
var adapterStopSignal = new EventWaitHandle(false,
EventResetMode.ManualReset, outputConfig.AdapterStopSignal);
// Run the query
query.Start();
adapterStopSignal.WaitOne();
query.Stop();
application.Delete();
server.Dispose();
}
/// <summary>
/// Creates and binds a query with a single input stream.
/// </summary>
private static Query CreateQuery(Application application, CepStream<EventType> streamDefinition, Uri inputAddress, Uri outputAddress)
{
QueryTemplate template = application.CreateQueryTemplate("QueryTemplate", null, streamDefinition);
QueryBinder binder = new QueryBinder(template);
InputAdapter inputAdapter = application.CreateInputAdapter<WcfInputAdapterFactory>("InputAdapter", null);
binder.BindProducer("Input", inputAdapter, inputAddress, EventShape.Point);
OutputAdapter outputAdapter = application.CreateOutputAdapter<WcfOutputAdapterFactory>("OutputAdapter", null);
binder.AddConsumer("Output", outputAdapter, outputAddress, EventShape.Point, StreamEventOrder.FullyOrdered);
Query query = application.CreateQuery("Query", null, binder);
return query;
}
private static Query CreateQuery(Application application, CepStream<BrandQuote> streamDefinition, BrandQuoteInputConfig inputConfig, BrandQuoteOutputConfig outputConfig)
{
var template = application.CreateQueryTemplate("QueryTemplate", null, streamDefinition);
var binder = new QueryBinder(template);
var inputAdapter = application.CreateInputAdapter<BrandQuoteInputAdapterFactory>("InputAdapter", null);
binder.BindProducer("Input", inputAdapter, inputConfig, EventShape.Point);
var outputAdapter = application.CreateOutputAdapter<BrandQuoteOutputAdapterFactory>("OutputAdapter", null);
binder.AddConsumer("Output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.FullyOrdered);
var query = application.CreateQuery("Query", null, binder);
return query;
}
public void TestTwitter()
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("twittertest");
var inputConfig = new TwitterConfig
{
Username = UserNameTbx.Text,
Password = PasswordTbx.Text
};
if (FilterModeRbn.Checked)
{
inputConfig.Mode = TwitterMode.Filter;
inputConfig.Parameters = "track=" + FilterParametersTbx.Text;
}
else if (SampleModeRbn.Checked)
{
inputConfig.Mode = TwitterMode.Sample;
}
var input = CepStream <Tweet> .Create("twitter", typeof(TwitterFactory), inputConfig, EventShape.Point);
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
// Create query and bind to the output adapter
query = input.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
application.Delete();
server.Dispose();
}
private static void BindAndRunQuery <TPayload>(string queryName, CepStream <TPayload> queryStream, EventShape outputEventShape, List <string> inputFields, List <string> outputFields)
{
var inputConfig = new CsvInputConfig
{
InputFileName = @"..\..\..\TollInput.txt",
Delimiter = new char[] { ',' },
BufferSize = 4096,
CtiFrequency = 1,
CultureName = "en-US",
Fields = inputFields,
NonPayloadFieldCount = 2,
StartTimePos = 1,
EndTimePos = 2
};
// The adapter recognizes empty filename as a write to console.
var outputConfig = new CsvOutputConfig
{
OutputFileName = string.Empty,
Delimiter = new string[] { "\t" },
CultureName = "en-US",
Fields = outputFields
};
// Note - Please change the instance name to the one you have chosen during installation
using (var server = Server.Create("Default"))
{
Application application = server.CreateApplication("TollStationApp");
// set up input and output adapters
InputAdapter inputAdapter = application.CreateInputAdapter <CsvInputFactory>("input", "Csv Input Source");
OutputAdapter outputAdapter = application.CreateOutputAdapter <CsvOutputFactory>("output", "Csv Output");
// set up the query template
QueryTemplate queryTemplate = application.CreateQueryTemplate("QueryTemplate", string.Empty, queryStream);
// set up advance time settings to enqueue CTIs
var advanceTimeGenerationSettings = new AdvanceTimeGenerationSettings(inputConfig.CtiFrequency, TimeSpan.Zero, true);
var advanceTimeSettings = new AdvanceTimeSettings(advanceTimeGenerationSettings, null, AdvanceTimePolicy.Adjust);
// Bind query template to input and output streams
QueryBinder queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <TollReading>("TollStream", inputAdapter, inputConfig, EventShape.Interval, advanceTimeSettings);
queryBinder.AddConsumer("outputStream", outputAdapter, outputConfig, outputEventShape, StreamEventOrder.FullyOrdered);
// Create a runnable query by binding the query template to the input stream of interval events,
// and to an output stream of fully ordered point events (through an output adapter instantiated
// from the output factory class)
Query query = application.CreateQuery(queryName, "Hello Toll Query", queryBinder);
RunQuery(query);
}
}
public static void Main()
{
var cepStream = CepStream <MediationData> .Create("cepStream1");
Console.WriteLine("\n-----------------------------Filter e.Number > 3\n");
var filter = from e in cepStream
where e.Number > 3
select e;
RunCepStream(filter);
Console.WriteLine("\n-----------------------------Get average for HoppingWindow(3,1)\n");
var avgCepStream = from w in cepStream.Where(e => true).HoppingWindow(TimeSpan.FromMinutes(3), TimeSpan.FromMinutes(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new MediationData()
{
Direction = "Average",
Number = w.Avg(e => e.Number)
};
RunCepStream(avgCepStream);
Console.WriteLine("\n-----------------------------Sum of TumblingWindow(3)\n");
var count = from win in cepStream.TumblingWindow(TimeSpan.FromMinutes(3))
select new MediationData()
{
Direction = "Average",
Number = win.Sum(e => e.Number)
};
RunCepStream(count);
Console.WriteLine("\n-----------------------------Group e.Direction With HoppingWindow(3,3)\n");
var groupCepStream = from e in cepStream
group e by e.Direction into eGroup
from w in eGroup.HoppingWindow(TimeSpan.FromMinutes(3), TimeSpan.FromMinutes(3), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new MediationData()
{
Direction = eGroup.Key,
Number = w.Avg(e => e.Number)
};
RunCepStream(groupCepStream);
Console.WriteLine("\n-----------------------------ShiftEvent by 7 min\n");
var shiftEventCepStream = from e in cepStream.ShiftEventTime(e => e.StartTime.AddMinutes(7))
select e;
RunCepStream(shiftEventCepStream);
application.Delete();
server.Dispose();
}
// Creates a binary stream of which windows exceed the threshold and
// returns true on the first timestamp of every Alpha epoch and false on every other occasion.
public static CepStream<double> detectBraid(CepStream<CsvReading2> inputStream)
{
// It contains the values from the PSD calculation.
var tmpStream = from e in inputStream
select e.Value1;
var tmpStream2 = detect(tmpStream);
//var tmpFlagString = from k in tmpStream2
// select FlagConversion(k);
return tmpStream2;
}
/// <summary>
/// Creates and binds a query with a single input stream.
/// </summary>
private static Query CreateQuery(Application application, CepStream <EventType> streamDefinition, Uri inputAddress, Uri outputAddress)
{
QueryTemplate template = application.CreateQueryTemplate("QueryTemplate", null, streamDefinition);
QueryBinder binder = new QueryBinder(template);
InputAdapter inputAdapter = application.CreateInputAdapter <WcfInputAdapterFactory>("InputAdapter", null);
binder.BindProducer("Input", inputAdapter, inputAddress, EventShape.Point);
OutputAdapter outputAdapter = application.CreateOutputAdapter <WcfOutputAdapterFactory>("OutputAdapter", null);
binder.AddConsumer("Output", outputAdapter, outputAddress, EventShape.Point, StreamEventOrder.FullyOrdered);
Query query = application.CreateQuery("Query", null, binder);
return(query);
}
public void TestFacebook()
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("facebooktest");
var inputConfig = new FacebookConfig
{
AccessToken = AccessTokenTbx.Text,
UsernameOrUniqueId = UsernameOrUniqueIdTbx.Text,
RefreshPeriod = int.Parse(RefreshIntervalTbx.Text) * 1000
};
var input = CepStream <FacebookItem> .Create("facebook", typeof(FacebookFactory), inputConfig, EventShape.Point);
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
// Create query and bind to the output adapter
var expr = from e in input
select new { e.Id, e.FromName, Message = !string.IsNullOrEmpty(e.Message) ? e.Message : e.Description };
query = expr.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.FullyOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
application.Delete();
server.Dispose();
}
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
var streamInsightInstanceName = Properties.Settings.Default.InstanceName;
var server = Server.Create(streamInsightInstanceName);
var application = server.CreateApplication("yahoofinancetest");
// Configure output adapter
var outputConfig = new StackerConfig
{
StackerCtlPipeName = stackerCtl1.PipeName,
StackerCtlHostName = "localhost"
};
var inputConfig = new YahooFinanceConfig
{
StockSymbol = "^GSPC",
StockName = "S&P 500",
Timeout = 10000,
Interval = 1000
};
var input = CepStream <StockQuote> .Create("yahoo", typeof(YahooFinanceFactory), inputConfig, EventShape.Point);
// Create query and bind to the output adapter
query = input.ToQuery(application, "query", "...", typeof(StackerFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
stopFlag = false;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while (!stopFlag && (string)diagnosticView["QueryState"] == "Running");
// Stop query
query.Stop();
Thread.Sleep(1000);
application.Delete();
server.Dispose();
}
/// <summary>
/// Example of using a user-defined filter
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createUserFilterExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var filteredCepStream = from e in input
where UserDefinedFilter(e.FieldID)
select e;
var queryTemplate = application.CreateQueryTemplate("userFilterExampleTemplate", "Description...", filteredCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("userFilterExampleQuery", "Description...", queryBinder);
return(query);
}
private static void runQuery(CepStream <StockQuote> cepStream)
{
var observer = new Observer();
// Create waithandle to signal when query is finished
var adapterStopSignal = new EventWaitHandle(false, EventResetMode.AutoReset, "StockInsightSignal");
// This starts the query
var disposable = cepStream.ToObservable <StockQuote>().Subscribe(observer);
// Wait for query to finish
adapterStopSignal.WaitOne();
// Write a blank line
Console.WriteLine();
// This line has been commented out because it hangs sometimes
disposable.Dispose();
}
/// <summary>[Q5] “Report the output whenever Toll Booth 2 has the same number
/// of vehicles as Toll Booth 1, every time a change occurs in either stream”</summary>
private static void RunJoinDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var slidingWindowQuery = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
var stream1 = from e in slidingWindowQuery
where e.TollId.Equals(((int)1).ToString())
select e;
var stream2 = from e in slidingWindowQuery
where e.TollId.Equals(((int)2).ToString())
select e;
var queryStream = from e1 in stream1
join e2 in stream2
on e1.VehicleCount equals e2.VehicleCount
select new TollCompare
{
TollId1 = e1.TollId,
TollId2 = e2.TollId,
VehicleCount = e1.VehicleCount
};
BindAndRunQuery(
"Join",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId1", "TollId2", "VehicleCount"
});
}
/// <summary>[Q12] “Over a one hour tumbling window, report all commercial vehicles
/// with tonnage greater than one ton (2K lbs), along with their arrival
/// times at the toll, and any charges due to weight violation. Overweight
/// charges during the rush hour (7am to 7pm) are double that of non-rush hours”</summary>
private static void RunUDODemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(TimeSpan.FromHours((double)1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select w.VehicleWeights();
BindAndRunQuery(
"UDO",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"LicensePlate", "Weight", "WeightCharge"
});
}
private static void RunFilterProject()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream
where w.TollId == "1"
select w;
BindAndRunQuery(
"Tumbling",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
});
}
/// <summary>
/// Bucket things based on position and take counts.
/// </summary>
/// <param name="app">The application that will host the query.</param>
/// <param name="queryName">The name of the query.</param>
/// <param name="queryDescription">The description of the query.</param>
/// <param name="inputFile">The file to read input from.</param>
/// <param name="outputPath">The path under which to put our output. This will go to a file of the form [app].[query].out</param>
/// <param name="eventFrequency">How often the input adapter should produce events. Zero will produce them as fast as possible.</param>
/// <returns>The new query.</returns>
private static Query CreateBucketQuery(Application app, string queryName, string queryDescription, string inputFile, string outputPath, TimeSpan eventFrequency)
{
// Create a configuration for our input adapter.
var inputConfig = new ReplayablePointCsvInputAdapterConfig(inputFile, eventFrequency);
// Declare the input stream.
var inputStream = CepStream <XYPayload> .Create(
queryName + ":input",
typeof(ReplayablePointCsvInputAdapterFactory),
inputConfig,
EventShape.Point,
null);
// Do some aggregation by leading digit of x and y coordinate.
var q = from e in inputStream
group e by new { XBucket = (int)(e.X * 10), YBucket = (int)(e.Y * 10) } into xygroups
from window in xygroups.TumblingWindow(TimeSpan.FromDays(1))
select new
{
XBucket = xygroups.Key.XBucket,
YBucket = xygroups.Key.YBucket,
Count = window.Count(),
AvgX = window.Avg(e => e.X),
AvgY = window.Avg(e => e.Y)
};
// Create the query. This declares which output adapter to use, and also configures
// the query for resiliency.
Query rval = q.ToQuery(
app,
queryName,
queryDescription,
typeof(DedupingPointCsvOuputAdapterFactory),
new DedupingPointCsvOutputAdapterConfig(Path.Combine(outputPath, GetOutputFileName(app.ShortName, queryName)), new string[] { "XBucket", "YBucket", "AvgX", "AvgY", "Count" }),
EventShape.Point,
StreamEventOrder.FullyOrdered,
true); // <== *** This says that the query is resilient and enables it for checkpointing. ***
return(rval);
}
/// <summary>
/// Compare USD and SEK stock prices using given exchange rate. Example of a cross join.
/// </summary>
/// <param name="application"></param>
/// <param name="ericUSDConfig"></param>
/// <param name="ericSEKConfig"></param>
/// <param name="USDSEKConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createCrossJoinExampleQuery(Application application, StockQuoteInputConfig ericUSDConfig, StockQuoteInputConfig ericSEKConfig, StockQuoteInputConfig USDSEKConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var ericUSDStream = CepStream <StockQuote> .Create("ericUSDStream");
var ericSEKStream = CepStream <StockQuote> .Create("ericSEKStream");
var USDSEKStream = CepStream <StockQuote> .Create("USDSEKStream");
var ericRecalcCepStream = from eUSD in ericUSDStream
from eXch in USDSEKStream // Cross join
where eUSD.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Recalc",
FieldID = "Close",
Value = eUSD.Value * eXch.Value // Convert ERIC USD quote to SEK
};
var ericCompareCepStream = from eRecalc in ericRecalcCepStream
from eSEK in ericSEKStream
where eSEK.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Compare",
FieldID = "Diff",
Value = eSEK.Value - eRecalc.Value
};
var queryTemplate = application.CreateQueryTemplate("ericCompareTemplate", "Description...", ericCompareCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("ericUSDStream", inputAdapter, ericUSDConfig, EventShape.Point);
queryBinder.BindProducer <StockQuote>("ericSEKStream", inputAdapter, ericSEKConfig, EventShape.Point);
queryBinder.BindProducer <StockQuote>("USDSEKStream", inputAdapter, USDSEKConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("ericCompareQuery", "Description...", queryBinder);
return(query);
}
public QueriesCore(string instance, bool distinctNeeded)
{
// TODO: Complete member initialization
this._distinctNeeded = distinctNeeded;
_cepServer = Server.Create(instance);
_cepApplication = _cepServer.CreateApplication("Monitor");
_pingSource = new InputCollection<PageRequest>();
_pingSourceGathering = new InputCollection<PageRequest>();
_pingInputStream = _pingSource.ToPointStream(
_cepApplication,
s => PointEvent.CreateInsert(DateTime.UtcNow, s),
AdvanceTimeSettings.IncreasingStartTime,
"PagePingStream");
_pingInputStreamGathering = _pingSourceGathering.ToPointStream(
_cepApplication,
s => PointEvent.CreateInsert(DateTime.UtcNow, s),
AdvanceTimeSettings.IncreasingStartTime,
"PagePingStream");
}
private static void RunQuery(CepStream <Payload> cepStream, Application application)
{
// Configure output adapter
// Create query and bind to the output adapter
var query = cepStream.ToQuery(application, Guid.NewGuid().ToString(), "description", typeof(PayloadOutputFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while ((string)diagnosticView[DiagnosticViewProperty.QueryState] == "Running");
// Stop query
query.Stop();
}
/// <summary>[tQ1] "Every 3 minutes, report the number of vehicles processed
/// that were being processed at some point during that period at
/// the toll station since the last result. Report the result at a
/// point in time, at the beginning of the 3 minute window"</summary>
private static void RunTumblingCountDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var queryStream = from w in inputStream.TumblingWindow(TimeSpan.FromMinutes(3), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new TollCount {
Count = w.Count()
};
BindAndRunQuery(
"Tumbling",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"Count"
});
}
public QueriesCore(string instance, bool distinctNeeded)
{
// TODO: Complete member initialization
this._distinctNeeded = distinctNeeded;
_cepServer = Server.Create(instance);
_cepApplication = _cepServer.CreateApplication("Monitor");
_pingSource = new InputCollection <PageRequest>();
_pingSourceGathering = new InputCollection <PageRequest>();
_pingInputStream = _pingSource.ToPointStream(
_cepApplication,
s => PointEvent.CreateInsert(DateTime.UtcNow, s),
AdvanceTimeSettings.IncreasingStartTime,
"PagePingStream");
_pingInputStreamGathering = _pingSourceGathering.ToPointStream(
_cepApplication,
s => PointEvent.CreateInsert(DateTime.UtcNow, s),
AdvanceTimeSettings.IncreasingStartTime,
"PagePingStream");
}
/// <summary>[Q8] “Report the top 2 toll amounts from the results of Q4 over a 3 minute tumbling window”</summary>
private static void RunTopKDemo()
{
var inputStream = CepStream <TollReading> .Create("TollStream");
var groupedSliding = from e in inputStream.AlterEventDuration(e => TimeSpan.FromMinutes((double)3))
group e by e.TollId into perTollBooth
from w in perTollBooth.SnapshotWindow(SnapshotWindowOutputPolicy.Clip)
select new Toll
{
TollId = perTollBooth.Key,
TollAmount = w.Sum(e => e.Toll),
VehicleCount = w.Count()
};
var queryStream = (from window in groupedSliding.TumblingWindow(TimeSpan.FromMinutes((double)3), HoppingWindowOutputPolicy.ClipToWindowEnd)
from e in window
orderby e.TollAmount descending
select e).Take(
(uint)2,
e => new TopEvents
{
TollRank = e.Rank,
TollAmount = e.Payload.TollAmount,
TollId = e.Payload.TollId,
VehicleCount = e.Payload.VehicleCount
});
BindAndRunQuery(
"Top-K",
queryStream,
EventShape.Interval,
new List <string>()
{
"TollId", "LicensePlate", "State", "Make", "Model", "VehicleType", "VehicleWeight", "Toll", "Tag"
},
new List <string>()
{
"TollRank", "TollAmount", "TollId", "VehicleCount"
});
}
/// <summary>
/// Example of using a user-defined aggregate
/// </summary>
/// <param name="application"></param>
/// <param name="inputConfig"></param>
/// <param name="outputConfig"></param>
/// <param name="inputAdapter"></param>
/// <param name="outputAdapter"></param>
/// <returns></returns>
private static Query createStandardDeviationExampleQuery(Application application, StockQuoteInputConfig inputConfig, StockQuoteOutputConfig outputConfig, InputAdapter inputAdapter, OutputAdapter outputAdapter)
{
var input = CepStream <StockQuote> .Create("input");
var stddevCepStream = from w in input.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC",
FieldID = "7-day Stddev",
Value = w.StandardDeviation()
};
var queryTemplate = application.CreateQueryTemplate("standardDeviationExampleTemplate", "Description...", stddevCepStream);
var queryBinder = new QueryBinder(queryTemplate);
queryBinder.BindProducer <StockQuote>("input", inputAdapter, inputConfig, EventShape.Point);
queryBinder.AddConsumer <StockQuote>("output", outputAdapter, outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
var query = application.CreateQuery("standardDeviationExampleQuery", "Description...", queryBinder);
return(query);
}
private static CepStream<double> detect(CepStream<double> inputStream)
{
return from win in inputStream.TumblingWindow<double>(TimeSpan.FromMinutes(6))
select win.BraidAgg<double>(e => e);
}
private static void runQuery(CepStream<StockQuote> cepStream)
{
var observer = new Observer();
// Create waithandle to signal when query is finished
var adapterStopSignal = new EventWaitHandle(false, EventResetMode.AutoReset, "StockInsightSignal");
// This starts the query
var disposable = cepStream.ToObservable<StockQuote>().Subscribe(observer);
// Wait for query to finish
adapterStopSignal.WaitOne();
// Write a blank line
Console.WriteLine();
// This line has been commented out because it hangs sometimes
disposable.Dispose();
}
/// <summary>
/// Example of utilising a user defined function for filtering
/// </summary>
/// <param name="ericUSDStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void userFilterExample(CepStream<StockQuote> cepStream, Application application)
{
var filteredCepStream = from e in cepStream
where UserDefinedFilter(e.FieldID)
select e;
runQuery(filteredCepStream, application);
}
/// <summary>
/// Example of using a custom aggregation function for standard deviation calculation.
/// </summary>
/// <param name="ericUSDStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void standardDeviationExample(CepStream<StockQuote> cepStream, Application application)
{
var stddevCepStream = from w in cepStream.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC",
FieldID = "7-day Stddev",
Value = w.StandardDeviation()
};
runQuery(stddevCepStream, application);
}
/// <summary>
/// Binds a stream to output adapter and runs the query
/// </summary>
/// <param name="cepStream"></param>
private static void runQuery(CepStream<StockQuote> cepStream, Application application)
{
// Configure output adapter
var outputConfig = new StockQuoteOutputConfig();
// Create query and bind to the output adapter
var query = cepStream.ToQuery(application, Guid.NewGuid().ToString(), "description", typeof(StockQuoteOutputFactory), outputConfig, EventShape.Point, StreamEventOrder.ChainOrdered);
// Start query
query.Start();
// Wait until query change state
DiagnosticView diagnosticView;
do
{
Thread.Sleep(100);
diagnosticView = query.Application.Server.GetDiagnosticView(query.Name);
} while ((string)diagnosticView[DiagnosticViewProperty.QueryState] == "Running");
// Stop query
query.Stop();
}
/// <summary>
/// Example of a grouping and average calculation on the groups
/// </summary>
/// <param name="ericUSDStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void groupApplyExample(CepStream<StockQuote> ericUSDStream, Application application)
{
var ericUSDGroupCepStream = from e in ericUSDStream
group e by e.FieldID into eGroup
from w in eGroup.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC 7-day avg",
FieldID = eGroup.Key,
Value = w.Avg(e => e.Value)
};
runQuery(ericUSDGroupCepStream, application);
}
/// <summary>
/// Example of filter function
/// </summary>
/// <param name="cepStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void filterExample(CepStream<StockQuote> cepStream, Application application)
{
// Return only "Close" values using a where-clause
var filteredCepStream = from e in cepStream
where e.FieldID == "Close"
select e;
runQuery(filteredCepStream, application);
}
/// <summary>
/// Example of cross join and projections
/// </summary>
/// <param name="ericSEKStream"></param>
/// <param name="ericUSDStream"></param>
/// <param name="USDSEKStream"></param>
private static void crossJoinProjectionExample(CepStream<StockQuote> ericSEKStream, CepStream<StockQuote> ericUSDStream, CepStream<StockQuote> USDSEKStream, Application application)
{
var ericRecalcCepStream = from eUSD in ericUSDStream
from eXch in USDSEKStream
where eUSD.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Recalc",
FieldID = "Close",
Value = eUSD.Value * eXch.Value // Convert ERIC USD quote to SEK
};
var ericCompareCepStream = from eRecalc in ericRecalcCepStream
from eSEK in ericSEKStream
where eSEK.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC-Compare",
FieldID = "Diff",
Value = eSEK.Value - eRecalc.Value
};
runQuery(ericCompareCepStream, application);
}
/// <summary>
/// Example of detecting price drops larger than 10% in 7 days
/// </summary>
/// <param name="ericUSDStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void bigLooserExample(CepStream<StockQuote> cepStream, Application application)
{
var bigLooserCepStream = (from e1 in cepStream
from e2 in cepStream.ShiftEventTime(e => e.StartTime.AddDays(7))
where e1.FieldID == "Close" && e2.FieldID == "Close"
select new StockQuote()
{
StockID = "ERIC > 10% drop",
FieldID = "Close",
Value = (e1.Value - e2.Value) / e2.Value * 100
}).Where(e => e.Value < -10);
runQuery(bigLooserCepStream, application);
}
/// <summary>
/// Example of window function
/// </summary>
/// <param name="cepStream"></param>
/// <param name="outputConfig"></param>
/// <param name="adapterStopSignal"></param>
private static void avgExample(CepStream<StockQuote> cepStream, Application application)
{
var avgCepStream = from w in cepStream.Where(e => e.FieldID == "Close")
.HoppingWindow(TimeSpan.FromDays(7), TimeSpan.FromDays(1), HoppingWindowOutputPolicy.ClipToWindowEnd)
select new StockQuote()
{
StockID = "ERIC",
FieldID = "7-day avg",
Value = w.Avg(e => e.Value)
};
runQuery(avgCepStream, application);
}