/// <summary>
/// Creates a new instance
/// </summary>
public LiveDataBasedDelistingEventProvider(SubscriptionDataConfig dataConfig, IDataQueueHandler dataQueueHandler)
{
_dataConfig = new SubscriptionDataConfig(dataConfig, typeof(Delisting));
_dataQueueHandler = dataQueueHandler;
_delistingEnumerator = dataQueueHandler.Subscribe(_dataConfig, (sender, args) =>
{
if (_delistingEnumerator != null && _delistingEnumerator.MoveNext())
{
var delisting = _delistingEnumerator.Current as Delisting;
if (delisting != null)
{
// we set the delisting date!
DelistingDate = new ReferenceWrapper <DateTime>(delisting.Time);
}
else
{
Log.Error($"LiveDataBasedDelistingEventProvider(): Current is null or not a {nameof(Delisting)} event: {_delistingEnumerator.Current?.GetType()}");
}
}
else
{
Log.Error("LiveDataBasedDelistingEventProvider(): new data available triggered with no data");
}
});
}
/// <summary>
/// Initializes this instance
/// </summary>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">The factor file provider to use</param>
/// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
/// <param name="startTime">Start date for the data request</param>
public override void Initialize(
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
IMapFileProvider mapFileProvider,
DateTime startTime)
{
base.Initialize(config, factorFileProvider, mapFileProvider, startTime);
_delistingEnumerator = _dataQueueHandler.Subscribe(_dataConfig, (sender, args) =>
{
if (_delistingEnumerator != null && _delistingEnumerator.MoveNext())
{
// live trading always returns true but could be null
if (_delistingEnumerator.Current != null)
{
var delisting = _delistingEnumerator.Current as Delisting;
if (delisting != null)
{
// we set the delisting date!
DelistingDate = new ReferenceWrapper <DateTime>(delisting.Time);
}
else
{
Log.Error($"LiveDataBasedDelistingEventProvider(): Current is not a {nameof(Delisting)} event: {_delistingEnumerator.Current?.GetType()}");
}
}
}
else
{
Log.Error("LiveDataBasedDelistingEventProvider(): new data available triggered with no data");
}
});
}
/// <summary>
/// Creates a new PaperTradingDataFeed for the algorithm/job
/// </summary>
/// <param name="algorithm">The algorithm to receive the data, used for a complete listing of active securities</param>
/// <param name="dataSource">Queable Source of the data</param>
/// <param name="job">The job being run</param>
public PaperTradingDataFeed(IAlgorithm algorithm, IDataQueueHandler dataSource, LiveNodePacket job)
: base(algorithm, dataSource)
{
_job = job;
_queue = dataSource;
// create a lookup keyed by SecurityType
var symbols = new Dictionary <SecurityType, List <string> >();
// Only subscribe equities and forex symbols
foreach (var security in algorithm.Securities.Values)
{
if (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
{
if (!symbols.ContainsKey(security.Type))
{
symbols.Add(security.Type, new List <string>());
}
symbols[security.Type].Add(security.Symbol);
}
}
// request for data from these symbols
_queue.Subscribe(job, symbols);
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
public void AddSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime)
{
var symbols = BuildTypeSymbolList(new[] { security });
_dataQueue.Subscribe(_job, symbols);
var subscription = CreateSubscription(_algorithm, _resultHandler, security, DateTime.UtcNow.ConvertFromUtc(_algorithm.TimeZone).Date, Time.EndOfTime);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
private IEnumerator <BaseData> Subscribe(IDataQueueHandler dataQueueHandler,
SubscriptionDataConfig dataConfig,
EventHandler newDataAvailableHandler)
{
var enumerator = dataQueueHandler.Subscribe(dataConfig, newDataAvailableHandler);
if (enumerator != null)
{
return(enumerator);
}
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="universe">The universe the subscription is to be added to</param>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <returns>True if the subscription was created and added successfully, false otherwise</returns>
public bool AddSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
// create and add the subscription to our collection
var subscription = CreateSubscription(universe, security, utcStartTime, utcEndTime);
// for some reason we couldn't create the subscription
if (subscription == null)
{
Log.Trace("Unable to add subscription for: " + security.Symbol);
return(false);
}
Log.Trace("LiveTradingDataFeed.AddSubscription(): Added " + security.Symbol);
_subscriptions[new SymbolSecurityType(subscription)] = subscription;
// send the subscription for the new symbol through to the data queuehandler
// unless it is custom data, custom data is retrieved using the same as backtest
if (!subscription.Configuration.IsCustomData)
{
_dataQueueHandler.Subscribe(_job, new Dictionary <SecurityType, List <string> >
{
{ security.Type, new List <string> {
security.Symbol
} }
});
}
// keep track of security changes, we emit these to the algorithm
// as notifications, used in universe selection
_changes += SecurityChanges.Added(security);
// update our fill forward resolution setting
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return(true);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue <List <BaseData> > [Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List <decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue <List <BaseData> >();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
_isActive = true;
_algorithm = algorithm;
_resultHandler = resultHandler;
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, true);
_dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
Bridge = new BusyBlockingCollection <TimeSlice>();
_subscriptions = new ConcurrentDictionary <SymbolSecurityType, LiveSubscription>();
var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7);
var periodEnd = Time.EndOfTime;
foreach (var security in algorithm.Securities.Values)
{
var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm.Securities.Values);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
Bridge = new BlockingCollection <TimeSlice>();
//Set Properties:
_isActive = true;
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new IEnumerator <BaseData> [Subscriptions.Count];
_realtimePrices = new List <decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
//This is quantconnect data source, store here for speed/ease of access
var security = algorithm.Securities[_subscriptions[i].Symbol];
_isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData);
// only make readers for custom data, live data will come through data queue handler
if (_isDynamicallyLoadedData[i])
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var periodStart = DateTime.Today.AddDays(-7);
var subscriptionDataReader = new SubscriptionDataReader(
_subscriptions[i],
security,
DataFeedEndpoint.LiveTrading,
periodStart,
DateTime.MaxValue,
resultHandler,
Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue)
);
// wrap the subscription data reader with a filter enumerator
_subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue);
}
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket) job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue, resultHandler);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
_isActive = true;
_algorithm = algorithm;
_resultHandler = resultHandler;
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, true);
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
Bridge = new BusyBlockingCollection<TimeSlice>();
_subscriptions = new ConcurrentDictionary<SymbolSecurityType, LiveSubscription>();
var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7);
var periodEnd = Time.EndOfTime;
foreach (var security in algorithm.Securities.Values)
{
var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm.Securities.Values);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
_cancellationTokenSource = new CancellationTokenSource();
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
Bridge = new BlockingCollection<TimeSlice>();
//Set Properties:
_isActive = true;
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new IEnumerator<BaseData>[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket) job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
//This is quantconnect data source, store here for speed/ease of access
var security = algorithm.Securities[_subscriptions[i].Symbol];
_isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData);
// only make readers for custom data, live data will come through data queue handler
if (_isDynamicallyLoadedData[i])
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var periodStart = DateTime.Today.AddDays(-7);
var subscriptionDataReader = new SubscriptionDataReader(
_subscriptions[i],
security,
periodStart,
DateTime.MaxValue,
resultHandler,
Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue),
true,
DateTime.Today
);
// wrap the subscription data reader with a filter enumerator
_subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue);
}
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
