/// <summary> /// Initializes a new instance of the <see cref="TextSubscriptionFactory"/> class /// </summary> /// <param name="config">The subscription's configuration</param> /// <param name="date">The date this factory was produced to read data for</param> /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param> public TextSubscriptionFactory(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { _date = date; _config = config; _isLiveMode = isLiveMode; _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[0]); }
public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully() { var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm"); qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm)); // Initialize contains the statements: // self.AddData(Nifty, "NIFTY") // self.AddData(CustomPythonData, "IBM", Resolution.Daily) qcAlgorithm.Initialize(); var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY"); Assert.IsNotNull(niftySubscription); var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type }); Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false)); var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "IBM"); Assert.IsNotNull(customDataSubscription); Assert.IsTrue(customDataSubscription.IsCustomData); Assert.AreEqual("custom_data.CustomPythonData", customDataSubscription.Type.ToString()); var customDataFactory = (BaseData)ObjectActivator.GetActivator(customDataSubscription.Type).Invoke(new object[] { customDataSubscription.Type }); Assert.DoesNotThrow(() => customDataFactory.GetSource(customDataSubscription, DateTime.UtcNow, false)); }
public AbstractFieldNameTranslator GetFieldNameTranslator(ISearchIndex index) { SolrFieldMap fieldMap = index.Configuration.FieldMap as SolrFieldMap; SolrIndexSchema schema = index.Schema as SolrIndexSchema; ISettings instance = index.Locator.GetInstance <ISettings>(); SolrIndexConfiguration configuration = index.Configuration as SolrIndexConfiguration; Assert.IsNotNull(fieldMap, "FieldMap is null."); Assert.IsNotNull(schema, "SolrSchema is null."); Assert.IsNotNull(instance, "Settings is null."); Assert.IsNotNull(configuration, "This constructor requires SolrIndexConfiguration."); TemplateFieldTypeResolverFactory typeResolverFactory = configuration.TemplateFieldTypeResolverFactory; Assert.IsNotNull(typeResolverFactory, "normalizerFactory is null."); TemplateFieldTypeResolver fieldTypeResolver = typeResolverFactory.Create(); Assert.IsNotNull(fieldTypeResolver, "FieldTypeResolver is null."); SolrFieldConfigurationResolver configurationResolver = new SolrFieldConfigurationResolver(fieldMap, schema, fieldTypeResolver); var ctor = typeof(SolrFieldNameTranslator).Assembly .GetType("Sitecore.ContentSearch.SolrProvider.FieldNames.CultureContextGuard") .GetConstructors().First(); ICultureContextGuard cultureContextGuard = ObjectActivator.GetActivator <ICultureContextGuard>(ctor).Invoke(); return(new SynthesisSolrFieldNameTranslator(fieldMap, schema, instance, configurationResolver, new ExtensionStripHelper(fieldMap, schema), typeResolverFactory, cultureContextGuard)); }
/// <summary> /// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class /// </summary> /// <param name="config">The subscription's configuration</param> /// <param name="date">The date this factory was produced to read data for</param> /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param> public ZipEntryNameSubscriptionDataSourceReader(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { _config = config; _date = date; _isLiveMode = isLiveMode; _factory = _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); }
/// <summary> /// Creates an enumerator to read the specified request. /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var config = request.Configuration; // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators // also provides some immediate fast-forward to handle spooling through remote files quickly var frontier = Ref.Create(DateTime.MinValue); var lastSourceRefreshTime = DateTime.MinValue; var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); // this is refreshing the enumerator stack for each new source var refresher = new RefreshEnumerator <BaseData>(() => { // rate limit the refresh of this enumerator stack var utcNow = _timeProvider.GetUtcNow(); var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment); if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls) { return(Enumerable.Empty <BaseData>().GetEnumerator()); } lastSourceRefreshTime = utcNow; var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date; var source = sourceFactory.GetSource(config, localDate, true); // fetch the new source and enumerate the data source reader var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate); if (SourceRequiresFastForward(source)) { // apply fast forward logic for file transport mediums var maximumDataAge = GetMaximumDataAge(config.Increment); enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge); } else { // rate limit calls to this enumerator stack enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls); } if (source.Format == FileFormat.Collection) { // unroll collections into individual data points after fast forward/rate limiting applied enumerator = enumerator.SelectMany(data => { var collection = data as BaseDataCollection; return(collection?.Data.GetEnumerator() ?? new List <BaseData> { data }.GetEnumerator()); }); } return(enumerator); }); // prevent calls to the enumerator stack if current is in the future var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider)); }
public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully() { var pythonPath = new System.IO.DirectoryInfo("RegressionAlgorithms"); Environment.SetEnvironmentVariable("PYTHONPATH", pythonPath.FullName); var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm"); // Initialize contains the statements: // self.AddData(Nifty, "NIFTY") // self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily) qcAlgorithm.Initialize(); var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY"); Assert.IsNotNull(niftySubscription); var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type }); Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false)); var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "SCF/CME_CL1_ON"); Assert.IsNotNull(quandlSubscription); var quandlFactory = (BaseData)ObjectActivator.GetActivator(quandlSubscription.Type).Invoke(new object[] { quandlSubscription.Type }); Assert.DoesNotThrow(() => quandlFactory.GetSource(quandlSubscription, DateTime.UtcNow, false)); }
/// <summary> /// Initializes a new instance of the <see cref="CollectionSubscriptionDataSourceReader"/> class /// </summary> /// <param name="dataCacheProvider">Used to cache data for requested from the IDataProvider</param> /// <param name="config">The subscription's configuration</param> /// <param name="date">The date this factory was produced to read data for</param> /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param> public CollectionSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { _dataCacheProvider = dataCacheProvider; _date = date; _config = config; _isLiveMode = isLiveMode; _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); }
/// <summary> /// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class /// </summary> /// <param name="dataCacheProvider">This provider caches files if needed</param> /// <param name="config">The subscription's configuration</param> /// <param name="date">The date this factory was produced to read data for</param> /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param> public TextSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { _dataCacheProvider = dataCacheProvider; _date = date; _config = config; _isLiveMode = isLiveMode; _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); _shouldCacheDataPoints = !_config.IsCustomData && _config.Resolution >= Resolution.Hour && _config.Type != typeof(FineFundamental) && _config.Type != typeof(CoarseFundamental); }
/// <summary> /// Enumerate over the tick zip file and return a list of BaseData. /// </summary> /// <returns>IEnumerable of ticks</returns> public IEnumerable <BaseData> Parse() { var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]); ZipFile zipFile; using (var unzipped = Compression.Unzip(_zipPath, out zipFile)) { string line; while ((line = unzipped.ReadLine()) != null) { yield return(factory.Reader(_config, line, _date, false)); } } zipFile.Dispose(); }
/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var sourceFactory = (BaseData)ObjectActivator.GetActivator(request.Configuration.Type).Invoke(new object[] { request.Configuration.Type }); using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider)) { foreach (var date in _tradableDaysProvider(request)) { request.Configuration.MappedSymbol = GetMappedSymbol(request, date); var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode); var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode); var entriesForDate = factory.Read(source); foreach (var entry in entriesForDate) { yield return(entry); } } } }
public static object Clone(object instanceToClone) { if (instanceToClone == null) { return(null); } var type = instanceToClone.GetType(); var factory = ObjectActivator.GetActivator(type); var fields = GetFieldInfosIncludingBaseClasses(type, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); var instance = factory.Invoke(new object[0]); foreach (var field in fields) { field.SetValue(instance, field.GetValue(instanceToClone)); } return(instance); }
/// <summary> /// Enumerate over the tick zip file and return a list of BaseData. /// </summary> /// <returns>IEnumerable of ticks</returns> public IEnumerable <BaseData> Parse() { var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]); // for futures and options if no entry was provided we just read all if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType == SecurityType.Option || _config.SecurityType == SecurityType.FutureOption)) { foreach (var entries in Compression.Unzip(_zipPath)) { // we get the contract symbol from the zip entry var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key); foreach (var line in entries.Value) { var dataPoint = factory.Reader(_config, line, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } else { ZipFile zipFile; using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile)) { if (unzipped == null) { yield break; } string line; while ((line = unzipped.ReadLine()) != null) { yield return(factory.Reader(_config, line, _date, false)); } } zipFile.Dispose(); } }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="security">Security asset</param> /// <param name="feed">Feed type enum</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish) { //Save configuration of data-subscription: _config = config; //Save access to fill foward flag: _isFillForward = config.FillDataForward; //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; //Save access to securities _security = security; _isDynamicallyLoadedData = security.IsDynamicallyLoadedData; // do we have factor tables? _hasScaleFactors = SubscriptionAdjustment.HasScalingFactors(config.Symbol); //Save the type of data we'll be getting from the source. _feedEndpoint = feed; //Create the dynamic type-activators: _objectActivator = ObjectActivator.GetActivator(config.Type); if (_objectActivator == null) { Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }"); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = _objectActivator.Invoke(new object[] { }); _dataFactory = userObj as BaseData; //Save Access to the "Reader" Method: var readerMethod = _dataFactory.GetType().GetMethod("Reader", new[] { typeof(SubscriptionDataConfig), typeof(string), typeof(DateTime), typeof(DataFeedEndpoint) }); //Create a Delagate Accessor. _readerMethodInvoker = readerMethod.DelegateForCallMethod(); //Save access to the "GetSource" Method: _getSourceMethod = _dataFactory.GetType().GetMethod("GetSource", new[] { typeof(SubscriptionDataConfig), typeof(DateTime), typeof(DataFeedEndpoint) }); //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { quandl.SetAuthCode(Config.Get("quandl-auth-token")); } //Load the entire factor and symbol mapping tables into memory try { if (_hasScaleFactors) { _priceFactors = SubscriptionAdjustment.GetFactorTable(config.Symbol); _symbolMap = SubscriptionAdjustment.GetMapTable(config.Symbol); } } catch (Exception err) { Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message); _priceFactors = new SortedDictionary <DateTime, decimal>(); _symbolMap = new SortedDictionary <DateTime, string>(); } }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var objectActivator = ObjectActivator.GetActivator(config.Type); var sourceProvider = (BaseData)objectActivator.Invoke(new object[] { config.Type }); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, config, dateInDataTimeZone, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="security">Security asset</param> /// <param name="feed">Feed type enum</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish) { //Save configuration of data-subscription: _config = config; AuxiliaryData = new Queue <BaseData>(); //Save access to fill foward flag: _isFillForward = config.FillDataForward; //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; //Save access to securities _security = security; _isDynamicallyLoadedData = security.IsDynamicallyLoadedData; // do we have factor tables? _hasScaleFactors = FactorFile.HasScalingFactors(config.Symbol); //Save the type of data we'll be getting from the source. _feedEndpoint = feed; //Create the dynamic type-activators: _objectActivator = ObjectActivator.GetActivator(config.Type); if (_objectActivator == null) { Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }"); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = _objectActivator.Invoke(new object[] { }); _dataFactory = userObj as BaseData; //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { quandl.SetAuthCode(Config.Get("quandl-auth-token")); } //Load the entire factor and symbol mapping tables into memory, we'll start with some defaults _factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>()); _mapFile = new MapFile(config.Symbol, new List <MapFileRow>()); try { if (_hasScaleFactors) { _factorFile = FactorFile.Read(config.Symbol); _mapFile = MapFile.Read(config.Symbol); } } catch (Exception err) { Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message); } }
/// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> /// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param> /// <param name="mapFileResolver">Used for resolving the correct map files</param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="dataProvider">Used for getting files not present on disk</param> /// <param name="dataCacheProvider">Used for caching files</param> /// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param> /// <param name="isLiveMode">True if we're in live mode, false otherwise</param> /// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param> public SubscriptionDataReader(SubscriptionDataConfig config, DateTime periodStart, DateTime periodFinish, IResultHandler resultHandler, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IEnumerable <DateTime> tradeableDates, bool isLiveMode, IDataCacheProvider dataCacheProvider, bool includeAuxilliaryData = true) { //Save configuration of data-subscription: _config = config; _auxiliaryData = new Queue <BaseData>(); //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; _dataProvider = dataProvider; _dataCacheProvider = dataCacheProvider; //Save access to securities _isLiveMode = isLiveMode; _includeAuxilliaryData = includeAuxilliaryData; //Save the type of data we'll be getting from the source. //Create the dynamic type-activators: var objectActivator = ObjectActivator.GetActivator(config.Type); _resultHandler = resultHandler; _tradeableDates = tradeableDates.GetEnumerator(); if (objectActivator == null) { _resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }"); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = objectActivator.Invoke(new object[] { config.Type }); _dataFactory = userObj as BaseData; //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } // If Tiingo data, set the access token in data factory var tiingo = _dataFactory as TiingoDailyData; if (tiingo != null) { if (!Tiingo.IsAuthCodeSet) { Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } } _factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>()); _mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>()); // load up the map and factor files for equities if (!config.IsCustomData && config.SecurityType == SecurityType.Equity) { try { var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } var factorFile = factorFileProvider.Get(_config.Symbol); _hasScaleFactors = factorFile != null; if (_hasScaleFactors) { _factorFile = factorFile; // if factor file has minimum date, update start period if before minimum date if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue) { if (_periodStart < _factorFile.FactorFileMinimumDate.Value) { _periodStart = _factorFile.FactorFileMinimumDate.Value; _resultHandler.DebugMessage( string.Format("Data for symbol {0} has been limited due to numerical precision issues in the factor file. The starting date has been set to {1}.", config.Symbol.Value, _factorFile.FactorFileMinimumDate.Value.ToShortDateString())); } } } } catch (Exception err) { Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": "); } } // load up the map and factor files for underlying of equity option if (!config.IsCustomData && config.SecurityType == SecurityType.Option) { try { var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.Underlying.ID.Symbol, config.Symbol.Underlying.ID.Date); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } } catch (Exception err) { Log.Error(err, "Map Factors: " + config.Symbol.ID + ": "); } } // Estimate delisting date. switch (_config.Symbol.ID.SecurityType) { case SecurityType.Future: _delistingDate = _config.Symbol.ID.Date; break; case SecurityType.Option: _delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol); break; default: _delistingDate = _mapFile.DelistingDate; break; } _subscriptionFactoryEnumerator = ResolveDataEnumerator(true); }
/// <summary> /// Initializes the <see cref="SubscriptionDataReader"/> instance /// </summary> public void Initialize() { if (_initialized) { return; } //Save the type of data we'll be getting from the source. //Create the dynamic type-activators: var objectActivator = ObjectActivator.GetActivator(_config.Type); if (objectActivator == null) { OnInvalidConfigurationDetected( new InvalidConfigurationDetectedEventArgs( $"Custom data type \'{_config.Type.Name}\' missing parameterless constructor " + $"E.g. public {_config.Type.Name}() {{ }}")); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = objectActivator.Invoke(new object[] { _config.Type }); _dataFactory = userObj as BaseData; //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } // If Tiingo data, set the access token in data factory var tiingo = _dataFactory as TiingoDailyData; if (tiingo != null) { if (!Tiingo.IsAuthCodeSet) { Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } } _factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>()); _mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>()); // load up the map and factor files for equities if (!_config.IsCustomData && _config.SecurityType == SecurityType.Equity) { try { var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.ID.Symbol, _config.Symbol.ID.Date); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } var factorFile = _factorFileProvider.Get(_config.Symbol); _hasScaleFactors = factorFile != null; if (_hasScaleFactors) { _factorFile = factorFile; // if factor file has minimum date, update start period if before minimum date if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue) { if (_periodStart < _factorFile.FactorFileMinimumDate.Value) { _periodStart = _factorFile.FactorFileMinimumDate.Value; OnNumericalPrecisionLimited( new NumericalPrecisionLimitedEventArgs( $"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " + $"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}.")); } } } } catch (Exception err) { Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": "); } } // load up the map and factor files for underlying of equity option if (!_config.IsCustomData && _config.SecurityType == SecurityType.Option) { try { var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.Underlying.ID.Symbol, _config.Symbol.Underlying.ID.Date); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } } catch (Exception err) { Log.Error(err, "Map Factors: " + _config.Symbol.ID + ": "); } } // Estimate delisting date. switch (_config.Symbol.ID.SecurityType) { case SecurityType.Future: _delistingDate = _config.Symbol.ID.Date; break; case SecurityType.Option: _delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol); break; default: _delistingDate = _mapFile.DelistingDate; break; } _subscriptionFactoryEnumerator = ResolveDataEnumerator(true); _initialized = true; }
/// <summary> /// Creates an enumerator to read the specified request. /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var config = request.Configuration; // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators // also provides some immediate fast-forward to handle spooling through remote files quickly var frontier = Ref.Create(request.StartTimeLocal); var lastSourceRefreshTime = DateTime.MinValue; var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); // this is refreshing the enumerator stack for each new source var refresher = new RefreshEnumerator <BaseData>(() => { // rate limit the refresh of this enumerator stack var utcNow = _timeProvider.GetUtcNow(); var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment); if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls) { return(Enumerable.Empty <BaseData>().GetEnumerator()); } lastSourceRefreshTime = utcNow; var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date; var source = sourceFactory.GetSource(config, localDate, true); // fetch the new source and enumerate the data source reader var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate); if (SourceRequiresFastForward(source)) { // The FastForwardEnumerator implements these two features: // (1) make sure we never emit past data // (2) data filtering based on a maximum data age // For custom data we don't want feature (2) because we would reject data points emitted later // (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age. // apply fast forward logic for file transport mediums var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan); enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge); } else { // rate limit calls to this enumerator stack enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls); } if (source.Format == FileFormat.Collection) { // unroll collections into individual data points after fast forward/rate limiting applied enumerator = enumerator.SelectMany(data => { var collection = data as BaseDataCollection; return(collection?.Data.GetEnumerator() ?? new List <BaseData> { data }.GetEnumerator()); }); } return(enumerator); }); return(refresher); }
/// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="security">Security asset</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> /// <param name="resultHandler"></param> /// <param name="tradeableDates">Defines the dates for which we'll request data, in order</param> /// <param name="isLiveMode">True if we're in live mode, false otherwise</param> /// <param name="symbolResolutionDate">The date used to resolve the correct symbol</param> public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DateTime periodStart, DateTime periodFinish, IResultHandler resultHandler, IEnumerable <DateTime> tradeableDates, bool isLiveMode, DateTime?symbolResolutionDate ) { //Save configuration of data-subscription: _config = config; _auxiliaryData = new Queue <BaseData>(); //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; //Save access to securities _security = security; _isDynamicallyLoadedData = security.IsDynamicallyLoadedData; _isLiveMode = isLiveMode; //Save the type of data we'll be getting from the source. //Create the dynamic type-activators: var objectActivator = ObjectActivator.GetActivator(config.Type); _resultHandler = resultHandler; _tradeableDates = tradeableDates.GetEnumerator(); if (objectActivator == null) { _resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }"); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = objectActivator.Invoke(new object[] { }); _dataFactory = userObj as BaseData; //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } //Load the entire factor and symbol mapping tables into memory, we'll start with some defaults _factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>()); _mapFile = new MapFile(config.Symbol, new List <MapFileRow>()); try { // do we have map/factor tables? -- only applies to equities if (!security.IsDynamicallyLoadedData && security.Type == SecurityType.Equity) { // resolve the correct map file as of the date _mapFile = MapFile.ResolveMapFile(config.Symbol, config.Market, symbolResolutionDate); _hasScaleFactors = FactorFile.HasScalingFactors(_mapFile.EntitySymbol, config.Market); if (_hasScaleFactors) { _factorFile = FactorFile.Read(config.Symbol, config.Market); } } } catch (Exception err) { Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message); } _subscriptionFactoryEnumerator = ResolveDataEnumerator(true); }
/// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> /// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param> /// <param name="mapFileResolver">Used for resolving the correct map files</param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param> /// <param name="isLiveMode">True if we're in live mode, false otherwise</param> /// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param> public SubscriptionDataReader(SubscriptionDataConfig config, DateTime periodStart, DateTime periodFinish, IResultHandler resultHandler, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, IEnumerable <DateTime> tradeableDates, bool isLiveMode, bool includeAuxilliaryData = true) { //Save configuration of data-subscription: _config = config; _auxiliaryData = new Queue <BaseData>(); //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; //Save access to securities _isLiveMode = isLiveMode; _includeAuxilliaryData = includeAuxilliaryData; //Save the type of data we'll be getting from the source. //Create the dynamic type-activators: var objectActivator = ObjectActivator.GetActivator(config.Type); _resultHandler = resultHandler; _tradeableDates = tradeableDates.GetEnumerator(); if (objectActivator == null) { _resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }"); _endOfStream = true; return; } //Create an instance of the "Type": var userObj = objectActivator.Invoke(new object[] {}); _dataFactory = userObj as BaseData; //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } _factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>()); _mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>()); // load up the map and factor files for equities if (!config.IsCustomData && config.SecurityType == SecurityType.Equity) { try { var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } var factorFile = factorFileProvider.Get(_config.Symbol); _hasScaleFactors = factorFile != null; if (_hasScaleFactors) { _factorFile = factorFile; } } catch (Exception err) { Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": "); } } _subscriptionFactoryEnumerator = ResolveDataEnumerator(true); }
/// <summary> /// Enumerate over the tick zip file and return a list of BaseData. /// </summary> /// <returns>IEnumerable of ticks</returns> public IEnumerable <BaseData> Parse() { if (!File.Exists(_zipPath)) { Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}"); yield break; } var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]); if (_config.Type.ImplementsStreamReader()) { using (var zip = new ZipFile(_zipPath)) { foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase))) { // we get the contract symbol from the zip entry if not already provided with the zip entry var symbol = _config.Symbol; if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption())) { symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName); } using (var entryReader = new StreamReader(zipEntry.OpenReader())) { while (!entryReader.EndOfStream) { var dataPoint = factory.Reader(_config, entryReader, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } } } // for futures and options if no entry was provided we just read all else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption())) { foreach (var entries in Compression.Unzip(_zipPath)) { // we get the contract symbol from the zip entry var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key); foreach (var line in entries.Value) { var dataPoint = factory.Reader(_config, line, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } else { ZipFile zipFile; using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile)) { if (unzipped == null) { yield break; } string line; while ((line = unzipped.ReadLine()) != null) { yield return(factory.Reader(_config, line, _date, false)); } } zipFile.Dispose(); } }