/// <summary>
/// Initializes a new instance of the <see cref="CollectionSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">Used to cache data for requested from the IDataProvider</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public CollectionSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
: base(dataCacheProvider, isLiveMode)
{
_date = date;
_config = config;
_factory = _config.GetBaseDataInstance();
}
/// <summary>
/// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public ZipEntryNameSubscriptionDataSourceReader(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_config = config;
_date = date;
_isLiveMode = isLiveMode;
_factory = _factory = config.GetBaseDataInstance();
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">This provider caches files if needed</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
: base(dataCacheProvider, isLiveMode)
{
_date = date;
_config = config;
_factory = config.GetBaseDataInstance();
_shouldCacheDataPoints = !_config.IsCustomData && _config.Resolution >= Resolution.Hour &&
_config.Type != typeof(FineFundamental) && _config.Type != typeof(CoarseFundamental) &&
!DataCacheProvider.IsDataEphemeral;
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">This provider caches files if needed</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
: base(dataCacheProvider, isLiveMode)
{
_date = date;
_config = config;
_factory = config.GetBaseDataInstance();
_shouldCacheDataPoints = !_config.IsCustomData && _config.Resolution >= Resolution.Hour &&
_config.Type != typeof(FineFundamental) && _config.Type != typeof(CoarseFundamental) &&
!DataCacheProvider.IsDataEphemeral;
var method = _config.Type.GetMethod("Reader",
new[] { typeof(SubscriptionDataConfig), typeof(StreamReader), typeof(DateTime), typeof(bool) });
if (method != null && method.DeclaringType == _config.Type)
{
_implementsStreamReader = true;
}
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataCacheProvider">Used to cache data</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
ISubscriptionDataSourceReader reader;
TextSubscriptionDataSourceReader textReader = null;
switch (source.Format)
{
case FileFormat.Csv:
reader = textReader = new TextSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
break;
case FileFormat.Collection:
reader = new CollectionSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
break;
case FileFormat.ZipEntryName:
reader = new ZipEntryNameSubscriptionDataSourceReader(config, date, isLiveMode);
break;
case FileFormat.Index:
return(new IndexSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode));
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
// wire up event handlers for logging missing files
if (source.TransportMedium == SubscriptionTransportMedium.LocalFile)
{
var factory = config.GetBaseDataInstance();
if (!factory.IsSparseData())
{
reader.InvalidSource += (sender, args) => Log.Error($"SubscriptionDataSourceReader.InvalidSource(): File not found: {args.Source.Source}");
if (textReader != null)
{
textReader.CreateStreamReaderError += (sender, args) => Log.Error($"SubscriptionDataSourceReader.CreateStreamReaderError(): File not found: {args.Source.Source}");
}
}
}
return(reader);
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(_config.Symbol, exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoPrice;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyAPI data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyAPI;
if (energyInformation != null)
{
if (!USEnergyAPI.IsAuthCodeSet)
{
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
// If Fred data, set the access token in data factory
var fred = _dataFactory as FredApi;
if (fred != null)
{
if (!FredApi.IsAuthCodeSet)
{
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_dataFactory.RequiresMapping())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && !_config.SecurityType.IsOption())
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}, {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}]"));
}
}
}
if (_periodStart < mapFile.FirstDate)
{
_periodStart = mapFile.FirstDate;
OnStartDateLimited(
new StartDateLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}," +
$" {mapFile.FirstDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}]"));
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
_delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
UpdateDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
List <BaseData> cache;
_shouldCacheDataPoints = _shouldCacheDataPoints &&
// only cache local files
source.TransportMedium == SubscriptionTransportMedium.LocalFile;
var cacheItem = _shouldCacheDataPoints
? BaseDataSourceCache.GetCacheItem(source.Source + _config.Type) : null;
if (cacheItem == null)
{
cache = new List <BaseData>();
using (var reader = CreateStreamReader(source))
{
// if the reader doesn't have data then we're done with this subscription
if (reader == null || reader.EndOfStream)
{
OnCreateStreamReaderError(_date, source);
yield break;
}
if (_factory == null)
{
// only create a factory if the stream isn't null
_factory = _config.GetBaseDataInstance();
}
// while the reader has data
while (!reader.EndOfStream)
{
BaseData instance = null;
string line = null;
try
{
if (reader.StreamReader != null && _implementsStreamReader)
{
instance = _factory.Reader(_config, reader.StreamReader, _date, IsLiveMode);
}
else
{
// read a line and pass it to the base data factory
line = reader.ReadLine();
instance = _factory.Reader(_config, line, _date, IsLiveMode);
}
}
catch (Exception err)
{
OnReaderError(line ?? "StreamReader", err);
}
if (instance != null && instance.EndTime != default(DateTime))
{
if (_shouldCacheDataPoints)
{
cache.Add(instance);
}
else
{
yield return(instance);
}
}
else if (reader.ShouldBeRateLimited)
{
yield return(instance);
}
}
}
if (!_shouldCacheDataPoints)
{
yield break;
}
cacheItem = new CacheItem(source.Source + _config.Type, cache);
BaseDataSourceCache.Add(cacheItem, CachePolicy);
}
cache = cacheItem.Value as List <BaseData>;
if (cache == null)
{
throw new InvalidOperationException("CacheItem can not be cast into expected type. " +
$"Type is: {cacheItem.Value.GetType()}");
}
// Find the first data point 10 days (just in case) before the desired date
// and subtract one item (just in case there was a time gap and data.Time is after _date)
var frontier = _date.AddDays(-10);
var index = cache.FindIndex(data => data.Time > frontier);
index = index > 0 ? (index - 1) : 0;
foreach (var data in cache.Skip(index))
{
var clone = data.Clone();
clone.Symbol = _config.Symbol;
yield return(clone);
}
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoPrice;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyAPI data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyAPI;
if (energyInformation != null)
{
if (!USEnergyAPI.IsAuthCodeSet)
{
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_dataFactory.RequiresMapping())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && _config.SecurityType != SecurityType.Option)
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(
$"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " +
$"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}."));
}
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
List <BaseData> cache = null;
_shouldCacheDataPoints = _shouldCacheDataPoints &&
// only cache local files
source.TransportMedium == SubscriptionTransportMedium.LocalFile;
string cacheKey = null;
if (_shouldCacheDataPoints)
{
cacheKey = source.Source + _config.Type;
BaseDataSourceCache.TryGetValue(cacheKey, out cache);
}
if (cache == null)
{
cache = _shouldCacheDataPoints ? new List <BaseData>(30000) : null;
using (var reader = CreateStreamReader(source))
{
if (reader == null)
{
// if the reader doesn't have data then we're done with this subscription
yield break;
}
if (_factory == null)
{
// only create a factory if the stream isn't null
_factory = _config.GetBaseDataInstance();
}
// while the reader has data
while (!reader.EndOfStream)
{
BaseData instance = null;
string line = null;
try
{
if (reader.StreamReader != null && _implementsStreamReader)
{
instance = _factory.Reader(_config, reader.StreamReader, _date, IsLiveMode);
}
else
{
// read a line and pass it to the base data factory
line = reader.ReadLine();
instance = _factory.Reader(_config, line, _date, IsLiveMode);
}
}
catch (Exception err)
{
OnReaderError(line ?? "StreamReader", err);
}
if (instance != null && instance.EndTime != default(DateTime))
{
if (_shouldCacheDataPoints)
{
cache.Add(instance);
}
else
{
yield return(instance);
}
}
else if (reader.ShouldBeRateLimited)
{
yield return(instance);
}
}
}
if (!_shouldCacheDataPoints)
{
yield break;
}
lock (CacheKeys)
{
CacheKeys.Enqueue(cacheKey);
// we create a new dictionary, so we don't have to take locks when reading, and add our new item
var newCache = new Dictionary <string, List <BaseData> >(BaseDataSourceCache)
{
[cacheKey] = cache
};
if (BaseDataSourceCache.Count > CacheSize)
{
var removeCount = 0;
// we remove a portion of the first in entries
while (++removeCount < (CacheSize / 4))
{
newCache.Remove(CacheKeys.Dequeue());
}
// update the cache instance
BaseDataSourceCache = newCache;
}
else
{
// update the cache instance
BaseDataSourceCache = newCache;
}
}
}
if (cache == null)
{
throw new InvalidOperationException($"Cache should not be null. Key: {cacheKey}");
}
// Find the first data point 10 days (just in case) before the desired date
// and subtract one item (just in case there was a time gap and data.Time is after _date)
var frontier = _date.AddDays(-10);
var index = cache.FindIndex(data => data.Time > frontier);
index = index > 0 ? (index - 1) : 0;
foreach (var data in cache.Skip(index))
{
var clone = data.Clone();
clone.Symbol = _config.Symbol;
yield return(clone);
}
}