/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.Configuration.Type == typeof(QuoteBar))
{
enumerator = new QuoteBarFillForwardEnumerator(enumerator);
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan(), request.Configuration.DataTimeZone);
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return(enumerator);
}
private static LiveSubscription CreateSubscription(IAlgorithm algorithm,
IResultHandler resultHandler,
Security security,
DateTime periodStart,
DateTime periodEnd)
{
IEnumerator <BaseData> enumerator = null;
if (security.SubscriptionDataConfig.IsCustomData)
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var subscriptionDataReader = new SubscriptionDataReader(
security.SubscriptionDataConfig,
periodStart, Time.EndOfTime,
resultHandler,
Time.EachTradeableDay(algorithm.Securities.Values, periodStart, periodEnd),
true
);
// wrap the subscription data reader with a filter enumerator
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, periodEnd);
}
return(new LiveSubscription(security, enumerator, periodStart, periodEnd, true, false));
}
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
{
var config = security.SubscriptionDataConfig;
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
if (userDefined)
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
}
return(null);
}
var symbolResolutionDate = userDefined ? (DateTime?)null : start;
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
return(subscription);
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return(null);
}
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return(subscription);
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable <DateTime> tradeableDates,
bool useSubscriptionDataReader,
bool aggregate)
{
IEnumerator <BaseData> enumerator;
if (useSubscriptionDataReader)
{
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
}
else
{
var sourceFactory = (BaseData)Activator.CreateInstance(config.Type);
enumerator = (from date in tradeableDates
let source = sourceFactory.GetSource(config, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
let entriesForDate = factory.Read(source)
from entry in entriesForDate
select entry).GetEnumerator();
}
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (config.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return(enumerator);
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
protected IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward);
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security,
request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours);
}
return(enumerator);
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return(enumerator);
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable <DateTime> tradeableDates)
{
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (config.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return(enumerator);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
Bridge = new BlockingCollection <TimeSlice>();
//Set Properties:
_isActive = true;
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new IEnumerator <BaseData> [Subscriptions.Count];
_realtimePrices = new List <decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
//This is quantconnect data source, store here for speed/ease of access
var security = algorithm.Securities[_subscriptions[i].Symbol];
_isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData);
// only make readers for custom data, live data will come through data queue handler
if (_isDynamicallyLoadedData[i])
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var periodStart = DateTime.Today.AddDays(-7);
var subscriptionDataReader = new SubscriptionDataReader(
_subscriptions[i],
security,
DataFeedEndpoint.LiveTrading,
periodStart,
DateTime.MaxValue,
resultHandler,
Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue)
);
// wrap the subscription data reader with a filter enumerator
_subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue);
}
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
_cancellationTokenSource = new CancellationTokenSource();
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
IsActive = true;
_endOfBridge = new bool[_subscriptions];
SubscriptionReaders = new IEnumerator <BaseData> [_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
RealtimePrices = new List <decimal>(_subscriptions);
//Class Privates:
_algorithm = algorithm;
// find the minimum resolution, ignoring ticks
var fillForwardResolution = Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution.ToTimeSpan())
.DefaultIfEmpty(TimeSpan.FromSeconds(1))
.Min();
// figure out how many subscriptions are at the minimum resolution
var subscriptionsAtMinimumResolution =
(from sub in Subscriptions
where sub.Resolution == Subscriptions.Min(x => x.Resolution)
select sub).Count();
Bridge = new BlockingCollection <TimeSlice>(Math.Min(1000, 50000 / subscriptionsAtMinimumResolution));
for (var i = 0; i < _subscriptions; i++)
{
_endOfBridge[i] = false;
var config = Subscriptions[i];
var start = algorithm.StartDate;
var end = algorithm.EndDate;
var security = _algorithm.Securities[Subscriptions[i].Symbol];
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, security, DataFeedEndpoint.FileSystem, start, end, resultHandler, tradeableDates);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
FillForwardFrontiers[i] = new DateTime();
// prime the pump for iteration in Run
_endOfBridge[i] = !SubscriptionReaders[i].MoveNext();
if (_endOfBridge[i])
{
Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol);
}
}
}
