public void Setup()
{
_config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
_factorFile = FactorFile.Read(
_config.Symbol.Value, _config.Symbol.ID.Market);
_rawDataEnumerator = new RawDataEnumerator();
}
public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData)
{
var factorFileProvider = new Mock <IFactorFileProvider>();
var config = new SubscriptionDataConfig(typeOfConfig, _security.Symbol, Resolution.Hour, TimeZones.NewYork, TimeZones.NewYork, true, true, false);
factorFileProvider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, config.Market));
var totalPoints = 8;
var time = new DateTime(2010, 1, 1);
var enumerator = Enumerable.Range(0, totalPoints).Select(x => new Delisting {
Time = time.AddHours(x)
}).GetEnumerator();
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
new SubscriptionRequest(
false,
null,
_security,
config,
DateTime.UtcNow,
Time.EndOfTime
),
enumerator,
factorFileProvider.Object,
false);
// Test our subscription stream to see if it emits the aux data it should be filtered
// by the SubscriptionUtils produce function if the config isn't for a TradeBar
int dataReceivedCount = 0;
while (subscription.MoveNext())
{
dataReceivedCount++;
if (subscription.Current != null && subscription.Current.Data.DataType == MarketDataType.Auxiliary)
{
Assert.IsTrue(shouldReceiveAuxData);
}
}
// If it should receive aux data it should have emitted all points
// otherwise none should have been emitted
if (shouldReceiveAuxData)
{
Assert.AreEqual(totalPoints, dataReceivedCount);
}
else
{
Assert.AreEqual(0, dataReceivedCount);
}
}
public void FactorFiles_CanBeGenerated_Accurately()
{
// Arrange
var yahooEvents = _yahooDataDownloader.DownloadSplitAndDividendData(_symbol, Parse.DateTime("01/01/1970"), DateTime.MaxValue);
var filePath = LeanData.GenerateRelativeFactorFilePath(_symbol);
var tolerance = 0.00001m;
if (!File.Exists(filePath))
{
throw new ArgumentException("This test requires an already calculated factor file." +
"Try using one of the pre-existing factor files ");
}
var originalFactorFileInstance = FactorFile.Read(PermTick, Market);
// we limit events to the penultimate time in our factor file (last one is 2050)
var lastValidRow = originalFactorFileInstance.SortedFactorFileData.Reverse().Skip(1).First();
// Act
var newFactorFileInstance = _factorFileGenerator.CreateFactorFile(yahooEvents.Where(data => data.Time.AddDays(-1) <= lastValidRow.Key).ToList());
var earliestDate = originalFactorFileInstance.SortedFactorFileData.First().Key;
var latestDate = originalFactorFileInstance.SortedFactorFileData.Last().Key;
// Assert
Assert.AreEqual(originalFactorFileInstance.SortedFactorFileData.Count,
newFactorFileInstance.SortedFactorFileData.Count);
for (var i = earliestDate; i < latestDate; i = i.AddDays(1))
{
FactorFileRow expected = null;
FactorFileRow actual = null;
originalFactorFileInstance.SortedFactorFileData.TryGetValue(i, out expected);
newFactorFileInstance.SortedFactorFileData.TryGetValue(i, out actual);
if (expected == null || actual == null)
{
Assert.IsTrue(actual == null);
Assert.IsTrue(expected == null);
}
else
{
Assert.IsTrue(Math.Abs(expected.PriceFactor - actual.PriceFactor) < tolerance);
Assert.IsTrue(Math.Abs(expected.SplitFactor - actual.SplitFactor) < tolerance);
}
}
}
public void ThrowingEnumeratorStackDisposesOfSubscription()
{
var enumerator = new TestDataEnumerator {
MoveNextTrueCount = 10, ThrowException = true
};
var factorFileProfider = new Mock <IFactorFileProvider>();
factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market));
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
new SubscriptionRequest(
false,
null,
_security,
_config,
DateTime.UtcNow,
Time.EndOfTime
),
enumerator,
factorFileProfider.Object,
false);
var count = 0;
while (enumerator.MoveNextTrueCount != 9)
{
if (count++ > 100)
{
Assert.Fail("Timeout waiting for producer");
}
Thread.Sleep(1);
}
Assert.IsFalse(subscription.MoveNext());
Assert.IsTrue(subscription.EndOfStream);
// enumerator is disposed by the producer
count = 0;
while (!enumerator.Disposed)
{
if (count++ > 100)
{
Assert.Fail("Timeout waiting for producer");
}
Thread.Sleep(1);
}
}
public void SubscriptionIsDisposed()
{
var dataPoints = 10;
var enumerator = new TestDataEnumerator {
MoveNextTrueCount = dataPoints
};
var factorFileProfider = new Mock <IFactorFileProvider>();
factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market));
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
new SubscriptionRequest(
false,
null,
_security,
_config,
DateTime.UtcNow,
Time.EndOfTime
),
enumerator,
factorFileProfider.Object,
false);
var count = 0;
while (enumerator.MoveNextTrueCount > 8)
{
if (count++ > 100)
{
Assert.Fail($"Timeout waiting for producer. {enumerator.MoveNextTrueCount}");
}
Thread.Sleep(1);
}
subscription.DisposeSafely();
Assert.IsFalse(subscription.MoveNext());
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
bool includeAuxilliaryData = true
)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
_hasScaleFactors = FactorFile.HasScalingFactors(_mapFile.Permtick, config.Market);
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(_mapFile.Permtick, config.Market);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="feed">Feed type enum</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish)
{
//Save configuration of data-subscription:
_config = config;
AuxiliaryData = new Queue <BaseData>();
//Save access to fill foward flag:
_isFillForward = config.FillDataForward;
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
// do we have factor tables?
_hasScaleFactors = FactorFile.HasScalingFactors(config.Symbol);
//Save the type of data we'll be getting from the source.
_feedEndpoint = feed;
//Create the dynamic type-activators:
_objectActivator = ObjectActivator.GetActivator(config.Type);
if (_objectActivator == null)
{
Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = _objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol);
_mapFile = MapFile.Read(config.Symbol);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler"></param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="symbolResolutionDate">The date used to resolve the correct symbol</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
Security security,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
DateTime?symbolResolutionDate
)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
_isLiveMode = isLiveMode;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
// do we have map/factor tables? -- only applies to equities
if (!security.IsDynamicallyLoadedData && security.Type == SecurityType.Equity)
{
// resolve the correct map file as of the date
_mapFile = MapFile.ResolveMapFile(config.Symbol, config.Market, symbolResolutionDate);
_hasScaleFactors = FactorFile.HasScalingFactors(_mapFile.EntitySymbol, config.Market);
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol, config.Market);
}
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
private static FactorFile GetFactorFile(string permtick)
{
return(FactorFile.Read(permtick, QuantConnect.Market.USA));
}
