public FileSourceCSV( FileSourceFactory factory, DataFlowOpInitializeContext context, AdapterInputSource adapterInputSource, bool hasHeaderLine, bool hasTitleLine, int?numLoops, string[] propertyNames, string dateFormat) { _factory = factory; _adapterInputSource = adapterInputSource; _hasHeaderLine = hasHeaderLine; _hasTitleLine = hasTitleLine; _numLoops = numLoops; _dateFormat = dateFormat; _statementContext = context.AgentInstanceContext.StatementContext; // use event type's full list of properties if (!hasTitleLine) { if (propertyNames != null) { _parseMake = SetupProperties(false, propertyNames, factory.OutputEventType, _statementContext, dateFormat); } else { _parseMake = SetupProperties(false, factory.OutputEventType.PropertyNames, factory.OutputEventType, _statementContext, dateFormat); } } }
public void TestNestedProperties() { var container = ContainerExtensions.CreateDefaultContainer(); var configuration = new Configuration(container); configuration.AddEventType <Figure>(); var ep = EPServiceProviderManager.GetProvider(container, "testNestedProperties", configuration); var ul = new SupportUpdateListener(); ep.EPAdministrator.CreateEPL("select * from Figure").Events += ul.Update; var source = new AdapterInputSource("regression/nestedProperties.csv"); var spec = new CSVInputAdapterSpec(source, "Figure"); var adapter = new CSVInputAdapter(_container, ep, spec); adapter.Start(); Assert.IsTrue(ul.IsInvoked()); var e = ul.AssertOneGetNewAndReset(); var f = (Figure)e.Underlying; Assert.AreEqual(1, f.Point.X); }
public void TestNestedMapProperties() { var configuration = new Configuration(_container); var point = new Dictionary <string, object>(); point.Put("X", typeof(int)); point.Put("Y", typeof(int)); var figure = new Dictionary <string, object>(); figure.Put("Name", typeof(string)); figure.Put("Point", point); configuration.Common.AddEventType("Figure", figure); var runtime = EPRuntimeProvider.GetRuntime("testNestedMapProperties", configuration); var ul = new SupportUpdateListener(); var stmt = CompileUtil.CompileDeploy(runtime, "select * from Figure").Statements[0]; stmt.Events += ul.Update; var source = new AdapterInputSource(_container, "regression/nestedProperties.csv"); var spec = new CSVInputAdapterSpec(source, "Figure"); var adapter = new CSVInputAdapter(runtime, spec); adapter.Start(); Assert.IsTrue(ul.IsInvoked()); var e = ul.AssertOneGetNewAndReset(); Assert.AreEqual(1, e.Get("Point.X")); }
public void TestNestedProperties() { var container = ContainerExtensions.CreateDefaultContainer(); var configuration = new Configuration(container); configuration.Common.AddEventType(typeof(Figure)); var runtime = EPRuntimeProvider.GetRuntime("testNestedProperties", configuration); var ul = new SupportUpdateListener(); var stmt = CompileUtil.CompileDeploy(runtime, "select * from Figure").Statements[0]; stmt.Events += ul.Update; var source = new AdapterInputSource(_container, "regression/nestedProperties.csv"); var spec = new CSVInputAdapterSpec(source, "Figure"); var adapter = new CSVInputAdapter(runtime, spec); adapter.Start(); Assert.IsTrue(ul.IsInvoked()); var e = ul.AssertOneGetNewAndReset(); var f = (Figure)e.Underlying; Assert.AreEqual(1, f.Point.X); }
public void TestNestedMapProperties() { var configuration = new Configuration(); var point = new Dictionary <string, object>(); point.Put("X", typeof(int)); point.Put("Y", typeof(int)); var figure = new Dictionary <string, object>(); figure.Put("Name", typeof(string)); figure.Put("Point", point); configuration.AddEventType("Figure", figure); var ep = EPServiceProviderManager.GetProvider("testNestedMapProperties", configuration); var ul = new SupportUpdateListener(); ep.EPAdministrator.CreateEPL("select * from Figure").Events += ul.Update; var source = new AdapterInputSource("regression/nestedProperties.csv"); var spec = new CSVInputAdapterSpec(source, "Figure"); var adapter = new CSVInputAdapter(ep, spec); adapter.Start(); Assert.IsTrue(ul.IsInvoked()); var e = ul.AssertOneGetNewAndReset(); Assert.AreEqual(1, e.Get("Point.X")); }
/// <summary> /// Ctor. /// </summary> /// <param name="container">The container.</param> /// <param name="adapterInputSource">the source of the CSV file</param> /// <exception cref="ArgumentException">AdapterInputSource cannot be null</exception> /// <throws>EPException in case of errors in reading the CSV file</throws> public CSVReader(IContainer container, AdapterInputSource adapterInputSource) { if (adapterInputSource == null) { throw new ArgumentException("AdapterInputSource cannot be null"); } _source = new CSVSource(container, adapterInputSource); }
private int[] markData; // backing store for reading data when marked /// <summary> /// Ctor. /// </summary> /// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param> public CSVSource(AdapterInputSource source) { rMarkIndex = -1; // not reading wMarkIndex = -1; // not writing eMarkIndex = -1; markData = new int[2]; stream = source.GetAsStream(); if (stream == null) { reader = source.GetAsReader(); } this.source = source; }
private int[] _markData; // backing store for reading data when marked /// <summary> /// Ctor. /// </summary> /// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param> public CSVSource(AdapterInputSource source) { _rMarkIndex = -1; // not reading _wMarkIndex = -1; // not writing _eMarkIndex = -1; _markData = new int[2]; _stream = source.GetAsStream(); if (_stream == null) { _reader = source.GetAsReader(); } _source = source; }
private void TrySource(AdapterInputSource source) { var spec = new CSVInputAdapterSpec(source, "TypeC"); EPService = EPServiceProviderManager.GetProvider("testPlayFromInputStream", MakeConfig("TypeC")); EPService.Initialize(); InputAdapter feed = new CSVInputAdapter(EPService, spec); EPStatement stmt = EPService.EPAdministrator.CreateEPL("select * from TypeC.win:length(100)"); var listener = new SupportUpdateListener(); stmt.Events += listener.Update; feed.Start(); Assert.AreEqual(1, listener.GetNewDataList().Count); }
private void TrySource(AdapterInputSource source) { var spec = new CSVInputAdapterSpec(source, "TypeC"); _runtime = EPRuntimeProvider.GetRuntime("testPlayFromInputStream", MakeConfig("TypeC")); _runtime.Initialize(); InputAdapter feed = new CSVInputAdapter(_runtime, spec); var stmt = CompileUtil.CompileDeploy(_runtime, "select * from TypeC#length(100)").Statements[0]; var listener = new SupportUpdateListener(); stmt.Events += listener.Update; feed.Start(); Assert.AreEqual(1, listener.GetNewDataList().Count); }
/// <summary> /// Ctor. /// </summary> /// <param name="container">The container.</param> /// <param name="source">the AdapterInputSource from which to obtain the underlying resource</param> public CSVSource(IContainer container, AdapterInputSource source) { _container = container; _rMarkIndex = -1; // not reading _wMarkIndex = -1; // not writing _eMarkIndex = -1; _markData = new int[2]; _stream = source.GetAsStream(_container); if (_stream == null) { _reader = source.GetAsReader(); } this._source = source; }
/// <summary> /// Ctor for adapters that will be passed to an AdapterCoordinator. /// </summary> /// <param name="adapterInputSource">the parameters for this adapter</param> /// <param name="eventTypeName">the event type name that the input adapter generates events for</param> public CSVInputAdapter(AdapterInputSource adapterInputSource, string eventTypeName) : this(null, adapterInputSource, eventTypeName) { }
/// <summary> /// Ctor. /// </summary> /// <param name="runtime">provides the engine runtime and services</param> /// <param name="adapterInputSource">the source of the CSV file</param> /// <param name="eventTypeName">the name of the Map event to create from the CSV data</param> public CSVInputAdapter(EPRuntime runtime, AdapterInputSource adapterInputSource, string eventTypeName) : this(runtime, new CSVInputAdapterSpec(adapterInputSource, eventTypeName)) { }
public DataFlowOperator Operator(DataFlowOpInitializeContext context) { var container = context.Container; var adapterInputSourceValue = DataFlowParameterResolution.ResolveOptionalInstance <AdapterInputSource>( "adapterInputSource", AdapterInputSource, context); var fileName = DataFlowParameterResolution.ResolveWithDefault <string>( "file", File, null, context); if (adapterInputSourceValue == null) { if (fileName != null) { adapterInputSourceValue = new AdapterInputSource(container, new FileInfo(fileName)); } else { throw new EPException("Failed to find required parameter, either the file or the adapterInputSource parameter is required"); } } var formatValue = DataFlowParameterResolution.ResolveStringOptional("format", Format, context); switch (formatValue) { case null: case "csv": { var hasHeaderLineFlag = DataFlowParameterResolution.ResolveWithDefault <bool?>( "hasHeaderLine", HasHeaderLine, false, context); var hasTitleLineFlag = DataFlowParameterResolution.ResolveWithDefault <bool?>( "hasTitleLine", HasTitleLine, false, context); var numLoopsValue = DataFlowParameterResolution.ResolveWithDefault <int?>( "numLoops", NumLoops, null, context); var dateFormatValue = DataFlowParameterResolution.ResolveStringOptional( "dateFormat", DateFormat, context); return(new FileSourceCSV( this, context, adapterInputSourceValue, hasHeaderLineFlag ?? false, hasTitleLineFlag ?? false, numLoopsValue, PropertyNames, dateFormatValue)); } case "line": { var propertyNameLineValue = DataFlowParameterResolution.ResolveStringOptional("propertyNameLine", PropertyNameLine, context); var propertyNameFileValue = DataFlowParameterResolution.ResolveStringOptional("propertyNameFile", PropertyNameFile, context); return(new FileSourceLineUnformatted(this, context, adapterInputSourceValue, fileName, propertyNameLineValue, propertyNameFileValue)); } default: throw new ArgumentException("Unrecognized file format '" + formatValue + "'"); } }
public void TestCoordinated() { IDictionary <string, object> priceProps = new Dictionary <string, object>(); priceProps.Put("timestamp", typeof(long?)); priceProps.Put("symbol", typeof(string)); priceProps.Put("price", typeof(double?)); IDictionary <string, object> tradeProps = new Dictionary <string, object>(); tradeProps.Put("timestamp", typeof(long?)); tradeProps.Put("symbol", typeof(string)); tradeProps.Put("notional", typeof(double?)); var config = new Configuration(_container); config.Common.AddEventType("TradeEvent", tradeProps); config.Common.AddEventType("PriceEvent", priceProps); _runtime = EPRuntimeProvider.GetRuntime("testCoordinated", config); _runtime.Initialize(); _runtime.EventService.ClockExternal(); _runtime.EventService.AdvanceTime(0); var sourcePrices = new AdapterInputSource(_container, CSV_FILENAME_TIMESTAMPED_PRICES); var inputPricesSpec = new CSVInputAdapterSpec(sourcePrices, "PriceEvent"); inputPricesSpec.TimestampColumn = "timestamp"; inputPricesSpec.PropertyTypes = priceProps; var inputPrices = new CSVInputAdapter(inputPricesSpec); var sourceTrades = new AdapterInputSource(_container, CSV_FILENAME_TIMESTAMPED_TRADES); var inputTradesSpec = new CSVInputAdapterSpec(sourceTrades, "TradeEvent"); inputTradesSpec.TimestampColumn = "timestamp"; inputTradesSpec.PropertyTypes = tradeProps; var inputTrades = new CSVInputAdapter(inputTradesSpec); var stmtPrices = CompileDeploy(_runtime, "select symbol, price from PriceEvent#length(100)").Statements[0]; var listenerPrice = new SupportUpdateListener(); stmtPrices.Events += listenerPrice.Update; var stmtTrade = CompileDeploy(_runtime, "select symbol, notional from TradeEvent#length(100)").Statements[0]; var listenerTrade = new SupportUpdateListener(); stmtTrade.Events += listenerTrade.Update; AdapterCoordinator coordinator = new AdapterCoordinatorImpl(_runtime, true); coordinator.Coordinate(inputPrices); coordinator.Coordinate(inputTrades); coordinator.Start(); _runtime.EventService.AdvanceTime(400); Assert.IsFalse(listenerTrade.IsInvoked()); Assert.IsFalse(listenerPrice.IsInvoked()); // invoke read of events at 500 (see CSV) _runtime.EventService.AdvanceTime(1000); Assert.AreEqual(1, listenerTrade.GetNewDataList().Count); Assert.AreEqual(1, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); // invoke read of price events at 1500 (see CSV) _runtime.EventService.AdvanceTime(2000); Assert.AreEqual(0, listenerTrade.GetNewDataList().Count); Assert.AreEqual(1, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); // invoke read of trade events at 2500 (see CSV) _runtime.EventService.AdvanceTime(3000); Assert.AreEqual(1, listenerTrade.GetNewDataList().Count); Assert.AreEqual(0, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); }
/// <summary> /// Ctor. /// </summary> /// <param name="container">The container.</param> /// <param name="epService">provides the engine runtime and services</param> /// <param name="adapterInputSource">the source of the CSV file</param> /// <param name="eventTypeName">the name of the Map event to create from the CSV data</param> public CSVInputAdapter(IContainer container, EPServiceProvider epService, AdapterInputSource adapterInputSource, String eventTypeName) : this(container, epService, new CSVInputAdapterSpec(adapterInputSource, eventTypeName)) { }
/// <summary> /// Ctor for adapters that will be passed to an AdapterCoordinator. /// </summary> /// <param name="container">The container.</param> /// <param name="adapterInputSource">the parameters for this adapter</param> /// <param name="eventTypeName">the event type name that the input adapter generates events for</param> public CSVInputAdapter(IContainer container, AdapterInputSource adapterInputSource, String eventTypeName) : this(container, null, adapterInputSource, eventTypeName) { }
public FileSourceLineUnformatted( FileSourceFactory factory, DataFlowOpInitializeContext context, AdapterInputSource inputSource, string filenameOrUri, string propertyNameLine, string propertyNameFile) { _factory = factory; _inputSource = inputSource; _filenameOrUri = filenameOrUri; _propertyNameLine = propertyNameLine; _propertyNameFile = propertyNameFile; var outputEventType = factory.OutputEventType; var statementContext = context.AgentInstanceContext.StatementContext; if ((outputEventType.PropertyNames.Length != 1 || outputEventType.PropertyDescriptors[0].PropertyType != typeof(string)) && propertyNameLine == null) { throw new ArgumentException( "Expecting an output event type that has a single property that is of type string, or alternatively specify the 'propertyNameLine' parameter"); } if (outputEventType is ObjectArrayEventType && outputEventType.PropertyDescriptors.Count == 1) { _lineProcessor = new LineProcessorObjectArray(); } else { var propertyNameLineToUse = propertyNameLine; if (propertyNameLineToUse == null) { propertyNameLineToUse = outputEventType.PropertyDescriptors[0].PropertyName; } if (!outputEventType.IsProperty(propertyNameLineToUse)) { throw new EPException("Failed to find property name '" + propertyNameLineToUse + "' in type '" + outputEventType.Name + "'"); } Type propertyType; try { propertyType = outputEventType.GetPropertyType(propertyNameLineToUse); } catch (PropertyAccessException ex) { throw new EPException("Invalid property name '" + propertyNameLineToUse + "': " + ex.Message, ex); } if (propertyType != typeof(string)) { throw new EPException("Invalid property type for property '" + propertyNameLineToUse + "', expected a property of type String"); } var writeables = EventTypeUtility.GetWriteableProperties(outputEventType, false); IList <WriteablePropertyDescriptor> writeableList = new List <WriteablePropertyDescriptor>(); var writeableLine = EventTypeUtility.FindWritable(propertyNameLineToUse, writeables); if (writeableLine == null) { throw new EPException("Failed to find writable property property '" + propertyNameLineToUse + "', is the property read-only?"); } writeableList.Add(writeableLine); if (propertyNameFile != null) { var writeableFile = EventTypeUtility.FindWritable(propertyNameFile, writeables); if (writeableFile == null || writeableFile.PropertyType != typeof(string)) { throw new EPException("Failed to find writable String-type property '" + propertyNameFile + "', is the property read-only?"); } writeableList.Add(writeableFile); } EventBeanManufacturer manufacturer; try { var writables = writeableList.ToArray(); manufacturer = EventTypeUtility .GetManufacturer(outputEventType, writables, statementContext.ImportServiceRuntime, false, statementContext.EventTypeAvroHandler) .GetManufacturer(statementContext.EventBeanTypedEventFactory); } catch (EventBeanManufactureException e) { throw new EPException("Event type '" + outputEventType.Name + "' cannot be written to: " + e.Message, e); } _lineProcessor = new LineProcessorGeneralPurpose(manufacturer); } if (factory.OutputPortTypes.Length == 2) { _eofProcessor = GetBeginEndProcessor(context, 1); } else if (factory.OutputPortTypes.Length == 3) { _bofProcessor = GetBeginEndProcessor(context, 1); _eofProcessor = GetBeginEndProcessor(context, 2); } else if (factory.OutputPortTypes.Length > 3) { throw new EPException("Operator only allows up to 3 output ports"); } }
public void TestCoordinated() { IDictionary <String, Object> priceProps = new Dictionary <String, Object>(); priceProps.Put("timestamp", typeof(long?)); priceProps.Put("symbol", typeof(String)); priceProps.Put("price", typeof(double?)); IDictionary <String, Object> tradeProps = new Dictionary <String, Object>(); tradeProps.Put("timestamp", typeof(long?)); tradeProps.Put("symbol", typeof(String)); tradeProps.Put("notional", typeof(double?)); var config = new Configuration(); config.AddEventType("TradeEvent", tradeProps); config.AddEventType("PriceEvent", priceProps); EPService = EPServiceProviderManager.GetProvider("testCoordinated", config); EPService.Initialize(); EPService.EPRuntime.SendEvent(new TimerControlEvent(TimerControlEvent.ClockTypeEnum.CLOCK_EXTERNAL)); EPService.EPRuntime.SendEvent(new CurrentTimeEvent(0)); var sourcePrices = new AdapterInputSource(CSV_FILENAME_TIMESTAMPED_PRICES); var inputPricesSpec = new CSVInputAdapterSpec(sourcePrices, "PriceEvent"); inputPricesSpec.TimestampColumn = "timestamp"; inputPricesSpec.PropertyTypes = priceProps; var inputPrices = new CSVInputAdapter(inputPricesSpec); var sourceTrades = new AdapterInputSource(CSV_FILENAME_TIMESTAMPED_TRADES); var inputTradesSpec = new CSVInputAdapterSpec(sourceTrades, "TradeEvent"); inputTradesSpec.TimestampColumn = "timestamp"; inputTradesSpec.PropertyTypes = tradeProps; var inputTrades = new CSVInputAdapter(inputTradesSpec); EPStatement stmtPrices = EPService.EPAdministrator.CreateEPL("select symbol, price from PriceEvent.win:length(100)"); var listenerPrice = new SupportUpdateListener(); stmtPrices.Events += listenerPrice.Update; EPStatement stmtTrade = EPService.EPAdministrator.CreateEPL("select symbol, notional from TradeEvent.win:length(100)"); var listenerTrade = new SupportUpdateListener(); stmtTrade.Events += listenerTrade.Update; AdapterCoordinator coordinator = new AdapterCoordinatorImpl(EPService, true); coordinator.Coordinate(inputPrices); coordinator.Coordinate(inputTrades); coordinator.Start(); EPService.EPRuntime.SendEvent(new CurrentTimeEvent(400)); Assert.IsFalse(listenerTrade.IsInvoked()); Assert.IsFalse(listenerPrice.IsInvoked()); // invoke read of events at 500 (see CSV) EPService.EPRuntime.SendEvent(new CurrentTimeEvent(1000)); Assert.AreEqual(1, listenerTrade.GetNewDataList().Count); Assert.AreEqual(1, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); // invoke read of price events at 1500 (see CSV) EPService.EPRuntime.SendEvent(new CurrentTimeEvent(2000)); Assert.AreEqual(0, listenerTrade.GetNewDataList().Count); Assert.AreEqual(1, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); // invoke read of trade events at 2500 (see CSV) EPService.EPRuntime.SendEvent(new CurrentTimeEvent(3000)); Assert.AreEqual(1, listenerTrade.GetNewDataList().Count); Assert.AreEqual(0, listenerPrice.GetNewDataList().Count); listenerTrade.Reset(); listenerPrice.Reset(); }
/// <summary>Ctor.</summary> /// <param name="adapterInputSource">the source for the CSV data</param> /// <param name="eventTypeName">the name of the event type created from the CSV data</param> public CSVInputAdapterSpec(AdapterInputSource adapterInputSource, string eventTypeName) { AdapterInputSource = adapterInputSource; EventTypeName = eventTypeName; }