public void PythonGetCustomData()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from AlgorithmImports import *
def Test(slice):
data = slice.Get(Quandl)
return data").GetAttr("Test");
var quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10
};
var quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11
};
var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
var data = test(new PythonSlice(slice));
Assert.AreEqual(2, (int)data.Count);
Assert.AreEqual(10, (int)data[Symbols.SPY].Value);
Assert.AreEqual(11, (int)data[Symbols.AAPL].Value);
}
}
public void PythonGetCustomData_Iterate()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
from QuantConnect.Data.Custom import *
def Test(slice):
data = slice.Get(Quandl)
count = 0
for singleData in data:
count += 1
if count != 2:
raise Exception('Unexpected value')").GetAttr("Test");
var quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10
};
var quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11
};
var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
}
}
public void PythonGetAndSymbolTradeBar()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
from QuantConnect.Tests import *
from QuantConnect.Data.Market import *
def Test(slice):
data = slice.Get(TradeBar, Symbols.AAPL)
value = data.Value
if value != 9:
raise Exception('Unexpected value')").GetAttr("Test");
var TradeBarSpy = new TradeBar {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 8
};
var TradeBarAapl = new TradeBar {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9
};
var quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10
};
var quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11
};
var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy });
Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
}
}
public void PythonGetTradeBar()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
from QuantConnect.Data.Market import *
def Test(slice):
data = slice.Get(TradeBar)
return data").GetAttr("Test");
var TradeBarSpy = new TradeBar {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 8
};
var TradeBarAapl = new TradeBar {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9
};
var quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10
};
var quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11
};
var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy });
var data = test(new PythonSlice(slice));
Assert.AreEqual(2, (int)data.Count);
Assert.AreEqual(8, (int)data[Symbols.SPY].Value);
Assert.AreEqual(9, (int)data[Symbols.AAPL].Value);
}
}
public void AccessesByDataType()
{
var tradeBar = new TradeBar {
Symbol = Symbols.SPY, Time = DateTime.UtcNow
};
var quandl = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now
};
var quoteBar = new QuoteBar {
Symbol = Symbols.SPY, Time = DateTime.Now
};
var tick = new Tick(DateTime.Now, Symbols.SPY, 1.1m, 2.1m)
{
TickType = TickType.Trade
};
var split = new Split(Symbols.SPY, DateTime.UtcNow, 1, 1, SplitType.SplitOccurred);
var delisting = new Delisting(Symbols.SPY, DateTime.UtcNow, 1, DelistingType.Delisted);
var slice = new Slice(DateTime.UtcNow, new BaseData[] { quoteBar, tradeBar, quandl, tick, split, delisting });
Assert.AreEqual(slice.Get(typeof(TradeBar))[Symbols.SPY], tradeBar);
Assert.AreEqual(slice.Get(typeof(Quandl))[Symbols.SPY], quandl);
Assert.AreEqual(slice.Get(typeof(QuoteBar))[Symbols.SPY], quoteBar);
Assert.AreEqual(slice.Get(typeof(Tick))[Symbols.SPY], tick);
Assert.AreEqual(slice.Get(typeof(Split))[Symbols.SPY], split);
Assert.AreEqual(slice.Get(typeof(Delisting))[Symbols.SPY], delisting);
}
public void PythonEnumerationWorks()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
from QuantConnect.Data.Custom import *
def Test(slice):
for dataPoint in slice:
return dataPoint").GetAttr("Test");
var quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11
};
var slice = new Slice(DateTime.Now, new[] { quandlAapl });
var data = test(new PythonSlice(slice)) as PyObject;
var keyValuePair = data.As <KeyValuePair <Symbol, BaseData> >();
Assert.IsNotNull(keyValuePair);
Assert.AreEqual(11, keyValuePair.Value.Value);
}
}
/// <summary>
/// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
/// </summary>
/// <param name="data">Data.</param>
public void OnData(Quandl data)
{
if (!Portfolio.HoldStock)
{
SetHoldings(_crude, 1);
Debug(Time.ToString("u") + " Purchased Crude Oil: " + _crude);
}
}
public Downloader()
{
quandl = new Quandl("eJkRsvbRyE3xK1NbqYVf");
settings = new Dictionary<string, string>();
settings.Add("type", "data");
//settings.Add("trim_start", "2010-02-01");
//settings.Add("trim_end", "2010-04-28");
//settings.Add("transformation", "normalize");
settings.Add("sort_order", "asc");
}
/// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
public void OnData(Quandl data)
{
if (!Portfolio.HoldStock)
{
//Order function places trades: enter the string symbol and the quantity you want:
SetHoldings(_quandlCode, 1);
//Debug sends messages to the user console: "Time" is the algorithm time keeper object
Debug("Purchased " + _quandlCode + " >> " + Time.ToShortDateString());
}
}
public void AccessesCustomGenericallyByTypeOtherTypesPresent()
{
var tradeBar = new TradeBar {
Symbol = Symbols.SPY, Time = DateTime.Now
};
var quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now
};
Slice slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, tradeBar });
DataDictionary <Quandl> quandlData = slice.Get <Quandl>();
Assert.AreEqual(1, quandlData.Count);
}
public void AccessesCustomGenericallyByType()
{
Quandl quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now
};
Quandl quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now
};
Slice slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
DataDictionary <Quandl> quandlData = slice.Get <Quandl>();
Assert.AreEqual(2, quandlData.Count);
}
public void AccessesGenericallyByTypeAndSymbol()
{
Quandl quandlSpy = new Quandl {
Symbol = Symbols.SPY, Time = DateTime.Now
};
Quandl quandlAapl = new Quandl {
Symbol = Symbols.AAPL, Time = DateTime.Now
};
Slice slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
Quandl quandlData = slice.Get <Quandl>(Symbols.SPY);
Assert.AreEqual(quandlSpy, quandlData);
}
public void QuandlTest()
{
var result = Quandl.GetSP500IndexConstituentsAsync().Result;
Assert.Equal("Apple Inc.", result.First(r => r.Ticker == "AAPL").Name);
var result2 = Quandl.GetFuturesMetadataAsync().Result;
Assert.Equal("Mini European 3.5% Fuel Oil Barges FOB Rdam (Platts)", result2.First(r => r.Symbol == "0D").Name);
var result3 = Quandl.GetCommoditiesAsync().Result;
Assert.Equal("Milk, Non-Fat Dry, Chicago", result3.First(r => r.Code == "WSJ/MILK").Name);
var result4 = Quandl.GetISOCurrencyCodesAsync().Result;
Assert.Equal("Euro", result4.First(r => r.Country == "AUSTRIA").Currency);
}
/// <summary>
/// Custom data event handler:
/// </summary>
/// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
public void OnData(Quandl data)
{
// Wait for all indicators to fully initialize
if (_smaVIX.IsReady && _smaVXV.IsReady && _ratio.IsReady)
{
if (!Portfolio.Invested && _ratio > 1)
{
MarketOrder(_vix, 100);
}
else if (_ratio < 1)
{
Liquidate();
}
// plot all indicators
PlotIndicator("SMA", _smaVIX, _smaVXV);
PlotIndicator("Ratio", _ratio);
}
}
public void QuandlDownloadDoesNotThrow()
{
Quandl.SetAuthCode("WyAazVXnq7ATy_fefTqm");
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var data = new HistoryAlgorithm.QuandlFuture();
const string ticker = "CHRIS/CME_SP1";
var date = new DateTime(2018, 8, 31);
var config = new SubscriptionDataConfig(typeof(HistoryAlgorithm.QuandlFuture), Symbol.Create(ticker, SecurityType.Base, QuantConnect.Market.USA), Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true);
var source = data.GetSource(config, date, false);
var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider());
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data);
var rows = factory.Read(source).ToList();
Assert.IsTrue(rows.Count > 0);
}
static void Main(string[] args)
{
Quandl myQuandl = new Quandl();
// Add the required settings to pull down data:
Dictionary<string, string> settings = new Dictionary<string, string>();
settings.Add("collapse", "weekly");
settings.Add("trim_start", "2010-02-01");
settings.Add("trim_end", "2010-04-28");
settings.Add("transformation", "normalize");
settings.Add("sort_order", "asc");
// Fetch:
IList<CsvFinancialFormat> data = myQuandl.GetData<CsvFinancialFormat>("YAHOO/MX_IBM", settings); //"GOOG/NYSE_IBM"
// Debug Purposes Only
foreach (CsvFinancialFormat tick in data)
{
//Console.WriteLine(tick.Time.ToShortDateString() + " H: " + tick.High);
Console.WriteLine(tick.InputString);
}
//Pause
Console.ReadKey();
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(_config.Symbol, exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoPrice;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyAPI data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyAPI;
if (energyInformation != null)
{
if (!USEnergyAPI.IsAuthCodeSet)
{
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
// If Fred data, set the access token in data factory
var fred = _dataFactory as FredApi;
if (fred != null)
{
if (!FredApi.IsAuthCodeSet)
{
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_dataFactory.RequiresMapping())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && !_config.SecurityType.IsOption())
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}, {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}]"));
}
}
}
if (_periodStart < mapFile.FirstDate)
{
_periodStart = mapFile.FirstDate;
OnStartDateLimited(
new StartDateLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}," +
$" {mapFile.FirstDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}]"));
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
_delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
UpdateDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
});
enumerator = enqueable;
}
else
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.Type.Name)
{
case nameof(QuoteBar):
var quoteBarAggregator = new QuoteBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.Quote && !tick.Suspicious)
{
quoteBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
});
enumerator = quoteBarAggregator;
break;
case nameof(TradeBar):
var tradeBarAggregator = new TradeBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
request.Security.Exchange.TimeZone,
_timeProvider);
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
if (data.DataType == MarketDataType.Auxiliary)
{
auxDataEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
else
{
var tick = data as Tick;
if (tick?.TickType == TickType.Trade && !tick.Suspicious)
{
tradeBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
});
enumerator = request.Configuration.SecurityType == SecurityType.Equity
? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
: tradeBarAggregator;
break;
case nameof(OpenInterest):
var oiAggregator = new OpenInterestEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
case nameof(Tick):
default:
// tick or streaming custom data subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
var tick = data as Tick;
if (tick != null)
{
if (tick.TickType == request.Configuration.TickType)
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
if (tick.TickType != TickType.OpenInterest)
{
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
}
else
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
});
enumerator = tickEnumerator;
break;
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="config">The subscription config to be added</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator <BaseData> enumerator;
if (config.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(config.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
if (config.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Custom data event handler:
/// </summary>
/// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
public void OnData(Quandl data)
{
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="feed">Feed type enum</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish)
{
//Save configuration of data-subscription:
_config = config;
AuxiliaryData = new Queue <BaseData>();
//Save access to fill foward flag:
_isFillForward = config.FillDataForward;
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
// do we have factor tables?
_hasScaleFactors = FactorFile.HasScalingFactors(config.Symbol);
//Save the type of data we'll be getting from the source.
_feedEndpoint = feed;
//Create the dynamic type-activators:
_objectActivator = ObjectActivator.GetActivator(config.Type);
if (_objectActivator == null)
{
Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = _objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol);
_mapFile = MapFile.Read(config.Symbol);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(_config.Type);
if (objectActivator == null)
{
OnInvalidConfigurationDetected(
new InvalidConfigurationDetectedEventArgs(
$"Custom data type \'{_config.Type.Name}\' missing parameterless constructor " +
$"E.g. public {_config.Type.Name}() {{ }}"));
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { _config.Type });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!_config.IsCustomData && _config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.ID.Symbol, _config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(
$"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " +
$"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}."));
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
// load up the map and factor files for underlying of equity option
if (!_config.IsCustomData && _config.SecurityType == SecurityType.Option)
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol.Underlying.ID.Symbol, _config.Symbol.Underlying.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "Map Factors: " + _config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler"></param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="symbolResolutionDate">The date used to resolve the correct symbol</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
Security security,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
DateTime?symbolResolutionDate
)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isDynamicallyLoadedData = security.IsDynamicallyLoadedData;
_isLiveMode = isLiveMode;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
//Load the entire factor and symbol mapping tables into memory, we'll start with some defaults
_factorFile = new FactorFile(config.Symbol, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol, new List <MapFileRow>());
try
{
// do we have map/factor tables? -- only applies to equities
if (!security.IsDynamicallyLoadedData && security.Type == SecurityType.Equity)
{
// resolve the correct map file as of the date
_mapFile = MapFile.ResolveMapFile(config.Symbol, config.Market, symbolResolutionDate);
_hasScaleFactors = FactorFile.HasScalingFactors(_mapFile.EntitySymbol, config.Market);
if (_hasScaleFactors)
{
_factorFile = FactorFile.Read(config.Symbol, config.Market);
}
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, subscriptionDataEnumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.Type.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyInformation data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyInformation;
if (energyInformation != null)
{
if (!USEnergyInformation.IsAuthCodeSet)
{
USEnergyInformation.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_config.TickerShouldBeMapped())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && _config.SecurityType != SecurityType.Option)
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(
$"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " +
$"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}."));
}
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
_initialized = true;
}
public void HandlesCustomDataBars()
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
var quandl = new Quandl();
quandl.Reader(config, "date,open,high,low,close,settle", DateTime.UtcNow, false);
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToString("yyyy-MM-dd")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(quandl.Reader(config, line, DateTime.UtcNow.AddDays(i), false));
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
if (!TradingEconomicsCalendar.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration))
{
var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler);
var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler);
enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
bool includeAuxilliaryData = true)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] {});
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": ");
}
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="dataProvider">Used for getting files not present on disk</param>
/// <param name="dataCacheProvider">Used for caching files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
IDataCacheProvider dataCacheProvider,
bool includeAuxilliaryData = true)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
_dataProvider = dataProvider;
_dataCacheProvider = dataCacheProvider;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { config.Type });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
_resultHandler.DebugMessage(
string.Format("Data for symbol {0} has been limited due to numerical precision issues in the factor file. The starting date has been set to {1}.",
config.Symbol.Value,
_factorFile.FactorFileMinimumDate.Value.ToShortDateString()));
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": ");
}
}
// load up the map and factor files for underlying of equity option
if (!config.IsCustomData && config.SecurityType == SecurityType.Option)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.Underlying.ID.Symbol, config.Symbol.Underlying.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "Map Factors: " + config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
