/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionFactory"/> class
/// </summary>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionFactory(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_date = date;
_config = config;
_isLiveMode = isLiveMode;
_factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[0]);
}
public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(CustomPythonData, "IBM", Resolution.Daily)
qcAlgorithm.Initialize();
var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY");
Assert.IsNotNull(niftySubscription);
var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type });
Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false));
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "IBM");
Assert.IsNotNull(customDataSubscription);
Assert.IsTrue(customDataSubscription.IsCustomData);
Assert.AreEqual("custom_data.CustomPythonData", customDataSubscription.Type.ToString());
var customDataFactory = (BaseData)ObjectActivator.GetActivator(customDataSubscription.Type).Invoke(new object[] { customDataSubscription.Type });
Assert.DoesNotThrow(() => customDataFactory.GetSource(customDataSubscription, DateTime.UtcNow, false));
}
public AbstractFieldNameTranslator GetFieldNameTranslator(ISearchIndex index)
{
SolrFieldMap fieldMap = index.Configuration.FieldMap as SolrFieldMap;
SolrIndexSchema schema = index.Schema as SolrIndexSchema;
ISettings instance = index.Locator.GetInstance <ISettings>();
SolrIndexConfiguration configuration = index.Configuration as SolrIndexConfiguration;
Assert.IsNotNull(fieldMap, "FieldMap is null.");
Assert.IsNotNull(schema, "SolrSchema is null.");
Assert.IsNotNull(instance, "Settings is null.");
Assert.IsNotNull(configuration, "This constructor requires SolrIndexConfiguration.");
TemplateFieldTypeResolverFactory typeResolverFactory = configuration.TemplateFieldTypeResolverFactory;
Assert.IsNotNull(typeResolverFactory, "normalizerFactory is null.");
TemplateFieldTypeResolver fieldTypeResolver = typeResolverFactory.Create();
Assert.IsNotNull(fieldTypeResolver, "FieldTypeResolver is null.");
SolrFieldConfigurationResolver configurationResolver = new SolrFieldConfigurationResolver(fieldMap, schema, fieldTypeResolver);
var ctor = typeof(SolrFieldNameTranslator).Assembly
.GetType("Sitecore.ContentSearch.SolrProvider.FieldNames.CultureContextGuard")
.GetConstructors().First();
ICultureContextGuard cultureContextGuard = ObjectActivator.GetActivator <ICultureContextGuard>(ctor).Invoke();
return(new SynthesisSolrFieldNameTranslator(fieldMap, schema, instance, configurationResolver, new ExtensionStripHelper(fieldMap, schema), typeResolverFactory, cultureContextGuard));
}
/// <summary>
/// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public ZipEntryNameSubscriptionDataSourceReader(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_config = config;
_date = date;
_isLiveMode = isLiveMode;
_factory = _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(DateTime.MinValue);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(config.Increment);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
// prevent calls to the enumerator stack if current is in the future
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider));
}
public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var pythonPath = new System.IO.DirectoryInfo("RegressionAlgorithms");
Environment.SetEnvironmentVariable("PYTHONPATH", pythonPath.FullName);
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
qcAlgorithm.Initialize();
var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY");
Assert.IsNotNull(niftySubscription);
var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type });
Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false));
var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "SCF/CME_CL1_ON");
Assert.IsNotNull(quandlSubscription);
var quandlFactory = (BaseData)ObjectActivator.GetActivator(quandlSubscription.Type).Invoke(new object[] { quandlSubscription.Type });
Assert.DoesNotThrow(() => quandlFactory.GetSource(quandlSubscription, DateTime.UtcNow, false));
}
/// <summary>
/// Initializes a new instance of the <see cref="CollectionSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">Used to cache data for requested from the IDataProvider</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public CollectionSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataCacheProvider = dataCacheProvider;
_date = date;
_config = config;
_isLiveMode = isLiveMode;
_factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">This provider caches files if needed</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataCacheProvider = dataCacheProvider;
_date = date;
_config = config;
_isLiveMode = isLiveMode;
_factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
_shouldCacheDataPoints = !_config.IsCustomData && _config.Resolution >= Resolution.Hour &&
_config.Type != typeof(FineFundamental) && _config.Type != typeof(CoarseFundamental);
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable <BaseData> Parse()
{
var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, out zipFile))
{
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return(factory.Reader(_config, line, _date, false));
}
}
zipFile.Dispose();
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var sourceFactory = (BaseData)ObjectActivator.GetActivator(request.Configuration.Type).Invoke(new object[] { request.Configuration.Type });
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
foreach (var date in _tradableDaysProvider(request))
{
request.Configuration.MappedSymbol = GetMappedSymbol(request, date);
var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
}
public static object Clone(object instanceToClone)
{
if (instanceToClone == null)
{
return(null);
}
var type = instanceToClone.GetType();
var factory = ObjectActivator.GetActivator(type);
var fields = GetFieldInfosIncludingBaseClasses(type, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic);
var instance = factory.Invoke(new object[0]);
foreach (var field in fields)
{
field.SetValue(instance, field.GetValue(instanceToClone));
}
return(instance);
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable <BaseData> Parse()
{
var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
// for futures and options if no entry was provided we just read all
if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType == SecurityType.Option || _config.SecurityType == SecurityType.FutureOption))
{
foreach (var entries in Compression.Unzip(_zipPath))
{
// we get the contract symbol from the zip entry
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
foreach (var line in entries.Value)
{
var dataPoint = factory.Reader(_config, line, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
else
{
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
{
if (unzipped == null)
{
yield break;
}
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return(factory.Reader(_config, line, _date, false));
}
}
zipFile.Dispose();
}
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="feed">Feed type enum</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish)
{
//Save configuration of data-subscription:
_config = config;
//Save access to fill foward flag:
_isFillForward = config.FillDataForward;
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
// do we have factor tables?
_hasScaleFactors = SubscriptionAdjustment.HasScalingFactors(config.Symbol);
//Save the type of data we'll be getting from the source.
_feedEndpoint = feed;
//Create the dynamic type-activators:
_objectActivator = ObjectActivator.GetActivator(config.Type);
if (_objectActivator == null)
{
Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = _objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//Save Access to the "Reader" Method:
var readerMethod = _dataFactory.GetType().GetMethod("Reader", new[] { typeof(SubscriptionDataConfig), typeof(string), typeof(DateTime), typeof(DataFeedEndpoint) });
//Create a Delagate Accessor.
_readerMethodInvoker = readerMethod.DelegateForCallMethod();
//Save access to the "GetSource" Method:
_getSourceMethod = _dataFactory.GetType().GetMethod("GetSource", new[] { typeof(SubscriptionDataConfig), typeof(DateTime), typeof(DataFeedEndpoint) });
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
//Load the entire factor and symbol mapping tables into memory
try
{
if (_hasScaleFactors)
{
_priceFactors = SubscriptionAdjustment.GetFactorTable(config.Symbol);
_symbolMap = SubscriptionAdjustment.GetMapTable(config.Symbol);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
_priceFactors = new SortedDictionary <DateTime, decimal>();
_symbolMap = new SortedDictionary <DateTime, string>();
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var objectActivator = ObjectActivator.GetActivator(config.Type);
var sourceProvider = (BaseData)objectActivator.Invoke(new object[] { config.Type });
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, config, dateInDataTimeZone, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="feed">Feed type enum</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish)
{
//Save configuration of data-subscription:
_config = config;
AuxiliaryData = new Queue <BaseData>();
//Save access to fill foward flag:
_isFillForward = config.FillDataForward;
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
// do we have factor tables?
_hasScaleFactors = FactorFile.HasScalingFactors(config.Symbol);
//Save the type of data we'll be getting from the source.
_feedEndpoint = feed;
//Create the dynamic type-activators:
_objectActivator = ObjectActivator.GetActivator(config.Type);
if (_objectActivator == null)
{
Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = _objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol);
_mapFile = MapFile.Read(config.Symbol);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="dataProvider">Used for getting files not present on disk</param>
/// <param name="dataCacheProvider">Used for caching files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
IDataCacheProvider dataCacheProvider,
bool includeAuxilliaryData = true)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
_dataProvider = dataProvider;
_dataCacheProvider = dataCacheProvider;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { config.Type });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
_resultHandler.DebugMessage(
string.Format("Data for symbol {0} has been limited due to numerical precision issues in the factor file. The starting date has been set to {1}.",
config.Symbol.Value,
_factorFile.FactorFileMinimumDate.Value.ToShortDateString()));
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": ");
}
}
// load up the map and factor files for underlying of equity option
if (!config.IsCustomData && config.SecurityType == SecurityType.Option)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.Underlying.ID.Symbol, config.Symbol.Underlying.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "Map Factors: " + config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(_config.Type);
if (objectActivator == null)
{
OnInvalidConfigurationDetected(
new InvalidConfigurationDetectedEventArgs(
$"Custom data type \'{_config.Type.Name}\' missing parameterless constructor " +
$"E.g. public {_config.Type.Name}() {{ }}"));
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { _config.Type });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!_config.IsCustomData && _config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.ID.Symbol, _config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(
$"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " +
$"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}."));
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
// load up the map and factor files for underlying of equity option
if (!_config.IsCustomData && _config.SecurityType == SecurityType.Option)
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.Underlying.ID.Symbol, _config.Symbol.Underlying.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "Map Factors: " + _config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(request.StartTimeLocal);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// The FastForwardEnumerator implements these two features:
// (1) make sure we never emit past data
// (2) data filtering based on a maximum data age
// For custom data we don't want feature (2) because we would reject data points emitted later
// (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age.
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
return(refresher);
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler"></param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="symbolResolutionDate">The date used to resolve the correct symbol</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
Security security,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
DateTime?symbolResolutionDate
)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
_isLiveMode = isLiveMode;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
// do we have map/factor tables? -- only applies to equities
if (!security.IsDynamicallyLoadedData && security.Type == SecurityType.Equity)
{
// resolve the correct map file as of the date
_mapFile = MapFile.ResolveMapFile(config.Symbol, config.Market, symbolResolutionDate);
_hasScaleFactors = FactorFile.HasScalingFactors(_mapFile.EntitySymbol, config.Market);
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol, config.Market);
}
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
bool includeAuxilliaryData = true)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] {});
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": ");
}
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable <BaseData> Parse()
{
if (!File.Exists(_zipPath))
{
Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}");
yield break;
}
var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
if (_config.Type.ImplementsStreamReader())
{
using (var zip = new ZipFile(_zipPath))
{
foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase)))
{
// we get the contract symbol from the zip entry if not already provided with the zip entry
var symbol = _config.Symbol;
if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName);
}
using (var entryReader = new StreamReader(zipEntry.OpenReader()))
{
while (!entryReader.EndOfStream)
{
var dataPoint = factory.Reader(_config, entryReader, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
}
}
// for futures and options if no entry was provided we just read all
else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
foreach (var entries in Compression.Unzip(_zipPath))
{
// we get the contract symbol from the zip entry
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
foreach (var line in entries.Value)
{
var dataPoint = factory.Reader(_config, line, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
else
{
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
{
if (unzipped == null)
{
yield break;
}
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return(factory.Reader(_config, line, _date, false));
}
}
zipFile.Dispose();
}
}
