public FileSourceCSV(
FileSourceFactory factory,
DataFlowOpInitializeContext context,
AdapterInputSource adapterInputSource,
bool hasHeaderLine,
bool hasTitleLine,
int?numLoops,
string[] propertyNames,
string dateFormat)
{
_factory = factory;
_adapterInputSource = adapterInputSource;
_hasHeaderLine = hasHeaderLine;
_hasTitleLine = hasTitleLine;
_numLoops = numLoops;
_dateFormat = dateFormat;
_statementContext = context.AgentInstanceContext.StatementContext;
// use event type's full list of properties
if (!hasTitleLine)
{
if (propertyNames != null)
{
_parseMake = SetupProperties(false, propertyNames, factory.OutputEventType, _statementContext, dateFormat);
}
else
{
_parseMake = SetupProperties(false, factory.OutputEventType.PropertyNames, factory.OutputEventType, _statementContext, dateFormat);
}
}
}
public void TestNestedProperties()
{
var container = ContainerExtensions.CreateDefaultContainer();
var configuration = new Configuration(container);
configuration.AddEventType <Figure>();
var ep = EPServiceProviderManager.GetProvider(container, "testNestedProperties", configuration);
var ul = new SupportUpdateListener();
ep.EPAdministrator.CreateEPL("select * from Figure").Events += ul.Update;
var source = new AdapterInputSource("regression/nestedProperties.csv");
var spec = new CSVInputAdapterSpec(source, "Figure");
var adapter = new CSVInputAdapter(_container, ep, spec);
adapter.Start();
Assert.IsTrue(ul.IsInvoked());
var e = ul.AssertOneGetNewAndReset();
var f = (Figure)e.Underlying;
Assert.AreEqual(1, f.Point.X);
}
public void TestNestedMapProperties()
{
var configuration = new Configuration(_container);
var point = new Dictionary <string, object>();
point.Put("X", typeof(int));
point.Put("Y", typeof(int));
var figure = new Dictionary <string, object>();
figure.Put("Name", typeof(string));
figure.Put("Point", point);
configuration.Common.AddEventType("Figure", figure);
var runtime = EPRuntimeProvider.GetRuntime("testNestedMapProperties", configuration);
var ul = new SupportUpdateListener();
var stmt = CompileUtil.CompileDeploy(runtime, "select * from Figure").Statements[0];
stmt.Events += ul.Update;
var source = new AdapterInputSource(_container, "regression/nestedProperties.csv");
var spec = new CSVInputAdapterSpec(source, "Figure");
var adapter = new CSVInputAdapter(runtime, spec);
adapter.Start();
Assert.IsTrue(ul.IsInvoked());
var e = ul.AssertOneGetNewAndReset();
Assert.AreEqual(1, e.Get("Point.X"));
}
public void TestNestedProperties()
{
var container = ContainerExtensions.CreateDefaultContainer();
var configuration = new Configuration(container);
configuration.Common.AddEventType(typeof(Figure));
var runtime = EPRuntimeProvider.GetRuntime("testNestedProperties", configuration);
var ul = new SupportUpdateListener();
var stmt = CompileUtil.CompileDeploy(runtime, "select * from Figure").Statements[0];
stmt.Events += ul.Update;
var source = new AdapterInputSource(_container, "regression/nestedProperties.csv");
var spec = new CSVInputAdapterSpec(source, "Figure");
var adapter = new CSVInputAdapter(runtime, spec);
adapter.Start();
Assert.IsTrue(ul.IsInvoked());
var e = ul.AssertOneGetNewAndReset();
var f = (Figure)e.Underlying;
Assert.AreEqual(1, f.Point.X);
}
public void TestNestedMapProperties()
{
var configuration = new Configuration();
var point = new Dictionary <string, object>();
point.Put("X", typeof(int));
point.Put("Y", typeof(int));
var figure = new Dictionary <string, object>();
figure.Put("Name", typeof(string));
figure.Put("Point", point);
configuration.AddEventType("Figure", figure);
var ep = EPServiceProviderManager.GetProvider("testNestedMapProperties", configuration);
var ul = new SupportUpdateListener();
ep.EPAdministrator.CreateEPL("select * from Figure").Events += ul.Update;
var source = new AdapterInputSource("regression/nestedProperties.csv");
var spec = new CSVInputAdapterSpec(source, "Figure");
var adapter = new CSVInputAdapter(ep, spec);
adapter.Start();
Assert.IsTrue(ul.IsInvoked());
var e = ul.AssertOneGetNewAndReset();
Assert.AreEqual(1, e.Get("Point.X"));
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="adapterInputSource">the source of the CSV file</param>
/// <exception cref="ArgumentException">AdapterInputSource cannot be null</exception>
/// <throws>EPException in case of errors in reading the CSV file</throws>
public CSVReader(IContainer container, AdapterInputSource adapterInputSource)
{
if (adapterInputSource == null)
{
throw new ArgumentException("AdapterInputSource cannot be null");
}
_source = new CSVSource(container, adapterInputSource);
}
private int[] markData; // backing store for reading data when marked
/// <summary>
/// Ctor.
/// </summary>
/// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param>
public CSVSource(AdapterInputSource source)
{
rMarkIndex = -1; // not reading
wMarkIndex = -1; // not writing
eMarkIndex = -1;
markData = new int[2];
stream = source.GetAsStream();
if (stream == null)
{
reader = source.GetAsReader();
}
this.source = source;
}
private int[] _markData; // backing store for reading data when marked
/// <summary>
/// Ctor.
/// </summary>
/// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param>
public CSVSource(AdapterInputSource source)
{
_rMarkIndex = -1; // not reading
_wMarkIndex = -1; // not writing
_eMarkIndex = -1;
_markData = new int[2];
_stream = source.GetAsStream();
if (_stream == null)
{
_reader = source.GetAsReader();
}
_source = source;
}
private void TrySource(AdapterInputSource source)
{
var spec = new CSVInputAdapterSpec(source, "TypeC");
EPService = EPServiceProviderManager.GetProvider("testPlayFromInputStream", MakeConfig("TypeC"));
EPService.Initialize();
InputAdapter feed = new CSVInputAdapter(EPService, spec);
EPStatement stmt = EPService.EPAdministrator.CreateEPL("select * from TypeC.win:length(100)");
var listener = new SupportUpdateListener();
stmt.Events += listener.Update;
feed.Start();
Assert.AreEqual(1, listener.GetNewDataList().Count);
}
private void TrySource(AdapterInputSource source)
{
var spec = new CSVInputAdapterSpec(source, "TypeC");
_runtime = EPRuntimeProvider.GetRuntime("testPlayFromInputStream", MakeConfig("TypeC"));
_runtime.Initialize();
InputAdapter feed = new CSVInputAdapter(_runtime, spec);
var stmt = CompileUtil.CompileDeploy(_runtime, "select * from TypeC#length(100)").Statements[0];
var listener = new SupportUpdateListener();
stmt.Events += listener.Update;
feed.Start();
Assert.AreEqual(1, listener.GetNewDataList().Count);
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param>
public CSVSource(IContainer container, AdapterInputSource source)
{
_container = container;
_rMarkIndex = -1; // not reading
_wMarkIndex = -1; // not writing
_eMarkIndex = -1;
_markData = new int[2];
_stream = source.GetAsStream(_container);
if (_stream == null)
{
_reader = source.GetAsReader();
}
this._source = source;
}
/// <summary>
/// Ctor for adapters that will be passed to an AdapterCoordinator.
/// </summary>
/// <param name="adapterInputSource">the parameters for this adapter</param>
/// <param name="eventTypeName">the event type name that the input adapter generates events for</param>
public CSVInputAdapter(AdapterInputSource adapterInputSource, string eventTypeName)
: this(null, adapterInputSource, eventTypeName)
{
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="runtime">provides the engine runtime and services</param>
/// <param name="adapterInputSource">the source of the CSV file</param>
/// <param name="eventTypeName">the name of the Map event to create from the CSV data</param>
public CSVInputAdapter(EPRuntime runtime, AdapterInputSource adapterInputSource, string eventTypeName)
: this(runtime, new CSVInputAdapterSpec(adapterInputSource, eventTypeName))
{
}
public DataFlowOperator Operator(DataFlowOpInitializeContext context)
{
var container = context.Container;
var adapterInputSourceValue = DataFlowParameterResolution.ResolveOptionalInstance <AdapterInputSource>(
"adapterInputSource",
AdapterInputSource,
context);
var fileName = DataFlowParameterResolution.ResolveWithDefault <string>(
"file",
File,
null,
context);
if (adapterInputSourceValue == null)
{
if (fileName != null)
{
adapterInputSourceValue = new AdapterInputSource(container, new FileInfo(fileName));
}
else
{
throw new EPException("Failed to find required parameter, either the file or the adapterInputSource parameter is required");
}
}
var formatValue = DataFlowParameterResolution.ResolveStringOptional("format", Format, context);
switch (formatValue)
{
case null:
case "csv": {
var hasHeaderLineFlag = DataFlowParameterResolution.ResolveWithDefault <bool?>(
"hasHeaderLine",
HasHeaderLine,
false,
context);
var hasTitleLineFlag = DataFlowParameterResolution.ResolveWithDefault <bool?>(
"hasTitleLine",
HasTitleLine,
false,
context);
var numLoopsValue = DataFlowParameterResolution.ResolveWithDefault <int?>(
"numLoops",
NumLoops,
null,
context);
var dateFormatValue = DataFlowParameterResolution.ResolveStringOptional(
"dateFormat",
DateFormat,
context);
return(new FileSourceCSV(
this,
context,
adapterInputSourceValue,
hasHeaderLineFlag ?? false,
hasTitleLineFlag ?? false,
numLoopsValue,
PropertyNames,
dateFormatValue));
}
case "line": {
var propertyNameLineValue = DataFlowParameterResolution.ResolveStringOptional("propertyNameLine", PropertyNameLine, context);
var propertyNameFileValue = DataFlowParameterResolution.ResolveStringOptional("propertyNameFile", PropertyNameFile, context);
return(new FileSourceLineUnformatted(this, context, adapterInputSourceValue, fileName, propertyNameLineValue, propertyNameFileValue));
}
default:
throw new ArgumentException("Unrecognized file format '" + formatValue + "'");
}
}
public void TestCoordinated()
{
IDictionary <string, object> priceProps = new Dictionary <string, object>();
priceProps.Put("timestamp", typeof(long?));
priceProps.Put("symbol", typeof(string));
priceProps.Put("price", typeof(double?));
IDictionary <string, object> tradeProps = new Dictionary <string, object>();
tradeProps.Put("timestamp", typeof(long?));
tradeProps.Put("symbol", typeof(string));
tradeProps.Put("notional", typeof(double?));
var config = new Configuration(_container);
config.Common.AddEventType("TradeEvent", tradeProps);
config.Common.AddEventType("PriceEvent", priceProps);
_runtime = EPRuntimeProvider.GetRuntime("testCoordinated", config);
_runtime.Initialize();
_runtime.EventService.ClockExternal();
_runtime.EventService.AdvanceTime(0);
var sourcePrices = new AdapterInputSource(_container, CSV_FILENAME_TIMESTAMPED_PRICES);
var inputPricesSpec = new CSVInputAdapterSpec(sourcePrices, "PriceEvent");
inputPricesSpec.TimestampColumn = "timestamp";
inputPricesSpec.PropertyTypes = priceProps;
var inputPrices = new CSVInputAdapter(inputPricesSpec);
var sourceTrades = new AdapterInputSource(_container, CSV_FILENAME_TIMESTAMPED_TRADES);
var inputTradesSpec = new CSVInputAdapterSpec(sourceTrades, "TradeEvent");
inputTradesSpec.TimestampColumn = "timestamp";
inputTradesSpec.PropertyTypes = tradeProps;
var inputTrades = new CSVInputAdapter(inputTradesSpec);
var stmtPrices = CompileDeploy(_runtime, "select symbol, price from PriceEvent#length(100)").Statements[0];
var listenerPrice = new SupportUpdateListener();
stmtPrices.Events += listenerPrice.Update;
var stmtTrade = CompileDeploy(_runtime, "select symbol, notional from TradeEvent#length(100)").Statements[0];
var listenerTrade = new SupportUpdateListener();
stmtTrade.Events += listenerTrade.Update;
AdapterCoordinator coordinator = new AdapterCoordinatorImpl(_runtime, true);
coordinator.Coordinate(inputPrices);
coordinator.Coordinate(inputTrades);
coordinator.Start();
_runtime.EventService.AdvanceTime(400);
Assert.IsFalse(listenerTrade.IsInvoked());
Assert.IsFalse(listenerPrice.IsInvoked());
// invoke read of events at 500 (see CSV)
_runtime.EventService.AdvanceTime(1000);
Assert.AreEqual(1, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(1, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
// invoke read of price events at 1500 (see CSV)
_runtime.EventService.AdvanceTime(2000);
Assert.AreEqual(0, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(1, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
// invoke read of trade events at 2500 (see CSV)
_runtime.EventService.AdvanceTime(3000);
Assert.AreEqual(1, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(0, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="epService">provides the engine runtime and services</param>
/// <param name="adapterInputSource">the source of the CSV file</param>
/// <param name="eventTypeName">the name of the Map event to create from the CSV data</param>
public CSVInputAdapter(IContainer container, EPServiceProvider epService, AdapterInputSource adapterInputSource, String eventTypeName)
: this(container, epService, new CSVInputAdapterSpec(adapterInputSource, eventTypeName))
{
}
/// <summary>
/// Ctor for adapters that will be passed to an AdapterCoordinator.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="adapterInputSource">the parameters for this adapter</param>
/// <param name="eventTypeName">the event type name that the input adapter generates events for</param>
public CSVInputAdapter(IContainer container, AdapterInputSource adapterInputSource, String eventTypeName)
: this(container, null, adapterInputSource, eventTypeName)
{
}
public FileSourceLineUnformatted(
FileSourceFactory factory,
DataFlowOpInitializeContext context,
AdapterInputSource inputSource,
string filenameOrUri,
string propertyNameLine,
string propertyNameFile)
{
_factory = factory;
_inputSource = inputSource;
_filenameOrUri = filenameOrUri;
_propertyNameLine = propertyNameLine;
_propertyNameFile = propertyNameFile;
var outputEventType = factory.OutputEventType;
var statementContext = context.AgentInstanceContext.StatementContext;
if ((outputEventType.PropertyNames.Length != 1 || outputEventType.PropertyDescriptors[0].PropertyType != typeof(string)) &&
propertyNameLine == null)
{
throw new ArgumentException(
"Expecting an output event type that has a single property that is of type string, or alternatively specify the 'propertyNameLine' parameter");
}
if (outputEventType is ObjectArrayEventType && outputEventType.PropertyDescriptors.Count == 1)
{
_lineProcessor = new LineProcessorObjectArray();
}
else
{
var propertyNameLineToUse = propertyNameLine;
if (propertyNameLineToUse == null)
{
propertyNameLineToUse = outputEventType.PropertyDescriptors[0].PropertyName;
}
if (!outputEventType.IsProperty(propertyNameLineToUse))
{
throw new EPException("Failed to find property name '" + propertyNameLineToUse + "' in type '" + outputEventType.Name + "'");
}
Type propertyType;
try {
propertyType = outputEventType.GetPropertyType(propertyNameLineToUse);
}
catch (PropertyAccessException ex) {
throw new EPException("Invalid property name '" + propertyNameLineToUse + "': " + ex.Message, ex);
}
if (propertyType != typeof(string))
{
throw new EPException("Invalid property type for property '" + propertyNameLineToUse + "', expected a property of type String");
}
var writeables = EventTypeUtility.GetWriteableProperties(outputEventType, false);
IList <WriteablePropertyDescriptor> writeableList = new List <WriteablePropertyDescriptor>();
var writeableLine = EventTypeUtility.FindWritable(propertyNameLineToUse, writeables);
if (writeableLine == null)
{
throw new EPException("Failed to find writable property property '" + propertyNameLineToUse + "', is the property read-only?");
}
writeableList.Add(writeableLine);
if (propertyNameFile != null)
{
var writeableFile = EventTypeUtility.FindWritable(propertyNameFile, writeables);
if (writeableFile == null || writeableFile.PropertyType != typeof(string))
{
throw new EPException("Failed to find writable String-type property '" + propertyNameFile + "', is the property read-only?");
}
writeableList.Add(writeableFile);
}
EventBeanManufacturer manufacturer;
try {
var writables = writeableList.ToArray();
manufacturer = EventTypeUtility
.GetManufacturer(outputEventType, writables, statementContext.ImportServiceRuntime, false, statementContext.EventTypeAvroHandler)
.GetManufacturer(statementContext.EventBeanTypedEventFactory);
}
catch (EventBeanManufactureException e) {
throw new EPException("Event type '" + outputEventType.Name + "' cannot be written to: " + e.Message, e);
}
_lineProcessor = new LineProcessorGeneralPurpose(manufacturer);
}
if (factory.OutputPortTypes.Length == 2)
{
_eofProcessor = GetBeginEndProcessor(context, 1);
}
else if (factory.OutputPortTypes.Length == 3)
{
_bofProcessor = GetBeginEndProcessor(context, 1);
_eofProcessor = GetBeginEndProcessor(context, 2);
}
else if (factory.OutputPortTypes.Length > 3)
{
throw new EPException("Operator only allows up to 3 output ports");
}
}
public void TestCoordinated()
{
IDictionary <String, Object> priceProps = new Dictionary <String, Object>();
priceProps.Put("timestamp", typeof(long?));
priceProps.Put("symbol", typeof(String));
priceProps.Put("price", typeof(double?));
IDictionary <String, Object> tradeProps = new Dictionary <String, Object>();
tradeProps.Put("timestamp", typeof(long?));
tradeProps.Put("symbol", typeof(String));
tradeProps.Put("notional", typeof(double?));
var config = new Configuration();
config.AddEventType("TradeEvent", tradeProps);
config.AddEventType("PriceEvent", priceProps);
EPService = EPServiceProviderManager.GetProvider("testCoordinated", config);
EPService.Initialize();
EPService.EPRuntime.SendEvent(new TimerControlEvent(TimerControlEvent.ClockTypeEnum.CLOCK_EXTERNAL));
EPService.EPRuntime.SendEvent(new CurrentTimeEvent(0));
var sourcePrices = new AdapterInputSource(CSV_FILENAME_TIMESTAMPED_PRICES);
var inputPricesSpec = new CSVInputAdapterSpec(sourcePrices, "PriceEvent");
inputPricesSpec.TimestampColumn = "timestamp";
inputPricesSpec.PropertyTypes = priceProps;
var inputPrices = new CSVInputAdapter(inputPricesSpec);
var sourceTrades = new AdapterInputSource(CSV_FILENAME_TIMESTAMPED_TRADES);
var inputTradesSpec = new CSVInputAdapterSpec(sourceTrades, "TradeEvent");
inputTradesSpec.TimestampColumn = "timestamp";
inputTradesSpec.PropertyTypes = tradeProps;
var inputTrades = new CSVInputAdapter(inputTradesSpec);
EPStatement stmtPrices =
EPService.EPAdministrator.CreateEPL("select symbol, price from PriceEvent.win:length(100)");
var listenerPrice = new SupportUpdateListener();
stmtPrices.Events += listenerPrice.Update;
EPStatement stmtTrade =
EPService.EPAdministrator.CreateEPL("select symbol, notional from TradeEvent.win:length(100)");
var listenerTrade = new SupportUpdateListener();
stmtTrade.Events += listenerTrade.Update;
AdapterCoordinator coordinator = new AdapterCoordinatorImpl(EPService, true);
coordinator.Coordinate(inputPrices);
coordinator.Coordinate(inputTrades);
coordinator.Start();
EPService.EPRuntime.SendEvent(new CurrentTimeEvent(400));
Assert.IsFalse(listenerTrade.IsInvoked());
Assert.IsFalse(listenerPrice.IsInvoked());
// invoke read of events at 500 (see CSV)
EPService.EPRuntime.SendEvent(new CurrentTimeEvent(1000));
Assert.AreEqual(1, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(1, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
// invoke read of price events at 1500 (see CSV)
EPService.EPRuntime.SendEvent(new CurrentTimeEvent(2000));
Assert.AreEqual(0, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(1, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
// invoke read of trade events at 2500 (see CSV)
EPService.EPRuntime.SendEvent(new CurrentTimeEvent(3000));
Assert.AreEqual(1, listenerTrade.GetNewDataList().Count);
Assert.AreEqual(0, listenerPrice.GetNewDataList().Count);
listenerTrade.Reset();
listenerPrice.Reset();
}
/// <summary>Ctor.</summary>
/// <param name="adapterInputSource">the source for the CSV data</param>
/// <param name="eventTypeName">the name of the event type created from the CSV data</param>
public CSVInputAdapterSpec(AdapterInputSource adapterInputSource, string eventTypeName)
{
AdapterInputSource = adapterInputSource;
EventTypeName = eventTypeName;
}
