public override void OnSecuritiesChanged(SecurityChanges changes)
{
_changes = changes;
foreach (var security in changes.RemovedSecurities)
{
// liquidate securities that have been removed
if (security.Invested)
{
Liquidate(security.Symbol);
Log("Exit " + security.Symbol + " at " + security.Close);
}
}
foreach (var security in changes.AddedSecurities)
{
// enter short positions on new securities
if (!security.Invested && security.Close != 0)
{
var qty = CalculateOrderQuantity(security.Symbol, -0.25m);
MarketOnOpenOrder(security.Symbol, qty);
Log("Enter " + security.Symbol + " at " + security.Close);
}
}
}
/// <summary>
/// Initializes a new <see cref="TimeSlice"/> containing the specified data
/// </summary>
public TimeSlice(DateTime time, int dataPointCount, Slice slice, List<KeyValuePair<Security, List<BaseData>>> data, List<KeyValuePair<Cash, BaseData>> cashBookUpdateData, List<KeyValuePair<Security, BaseData>> securitiesUpdateData, List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>> consolidatorUpdateData, List<KeyValuePair<Security, List<BaseData>>> customData, SecurityChanges securityChanges)
{
Time = time;
Data = data;
Slice = slice;
CustomData = customData;
DataPointCount = dataPointCount;
CashBookUpdateData = cashBookUpdateData;
SecuritiesUpdateData = securitiesUpdateData;
ConsolidatorUpdateData = consolidatorUpdateData;
SecurityChanges = securityChanges;
}
/// <summary>
/// Event - v3.0 DATA EVENT HANDLER: (Pattern) Basic template for user to override for receiving all subscription data in a single event
/// </summary>
/// <code>
/// TradeBars bars = slice.Bars;
/// Ticks ticks = slice.Ticks;
/// TradeBar spy = slice["SPY"];
/// List{Tick} aaplTicks = slice["AAPL"]
/// Quandl oil = slice["OIL"]
/// dynamic anySymbol = slice[symbol];
/// DataDictionary{Quandl} allQuandlData = slice.Get{Quand}
/// Quandl oil = slice.Get{Quandl}("OIL")
/// </code>
/// <param name="slice">The current slice of data keyed by symbol string</param>
public override void OnData(Slice slice)
{
if (slice.Bars.Count == 0) return;
if (_changes == SecurityChanges.None) return;
// start fresh
Liquidate();
var percentage = 1m / slice.Bars.Count;
foreach (var tradeBar in slice.Bars.Values)
{
SetHoldings(tradeBar.Symbol, percentage);
}
// reset changes
_changes = SecurityChanges.None;
}
//Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
public void OnData(TradeBars data)
{
// if we have no changes, do nothing
if (_changes == SecurityChanges.None) return;
// liquidate removed securities
foreach (var security in _changes.RemovedSecurities)
{
if (security.Invested)
{
Liquidate(security.Symbol);
}
}
// we want 20% allocation in each security in our universe
foreach (var security in _changes.AddedSecurities)
{
SetHoldings(security.Symbol, 0.2m);
}
_changes = SecurityChanges.None;
}
public void OnData(TradeBars data)
{
if (_changes == SecurityChanges.None) return;
// liquidate securities that fell out of our universe
foreach (var security in _changes.RemovedSecurities)
{
if (security.Invested)
{
Liquidate(security.Symbol);
}
}
// invest in securities just added to our universe
foreach (var security in _changes.AddedSecurities)
{
if (!security.Invested)
{
MarketOrder(security.Symbol, 10);
}
}
_changes = SecurityChanges.None;
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <param name="isUserDefinedSubscription">Set to true to prevent coarse universe selection from removing this subscription</param>
public void AddSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
var subscription = CreateSubscription(_resultHandler, security, utcStartTime, utcEndTime, security.SubscriptionDataConfig.Resolution, isUserDefinedSubscription);
if (subscription == null)
{
// subscription will be null when there's no tradeable dates for the security between the requested times, so
// don't even try to load the data
return;
}
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
// prime the pump, run method checks current before move next calls
PrimeSubscriptionPump(subscription, true);
_changes += new SecurityChanges(new List<Security> {security}, new List<Security>());
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="configuration">The configuration of the subscription to remove</param>
/// <returns>True if the subscription was successfully removed, false otherwise</returns>
public bool RemoveSubscription(SubscriptionDataConfig configuration)
{
// remove the subscription from our collection
Subscription subscription;
if (!_subscriptions.TryRemove(configuration, out subscription))
{
Log.Error("LiveTradingDataFeed.RemoveSubscription(): Unable to remove: " + configuration.ToString());
return false;
}
var security = subscription.Security;
// remove the subscriptions
if (subscription.Configuration.IsCustomData)
{
_customExchange.RemoveEnumerator(security.Symbol);
_customExchange.RemoveDataHandler(security.Symbol);
}
else
{
_dataQueueHandler.Unsubscribe(_job, new[] { security.Symbol });
_exchange.RemoveDataHandler(security.Symbol);
}
subscription.Dispose();
// keep track of security changes, we emit these to the algorithm
// as notications, used in universe selection
_changes += SecurityChanges.Removed(security);
Log.Trace("LiveTradingDataFeed.RemoveSubscription(): Removed " + configuration);
UpdateFillForwardResolution();
return true;
}
/// <summary>
/// Event fired each time the we add/remove securities from the data feed
/// </summary>
/// <param name="changes">Object containing AddedSecurities and RemovedSecurities</param>
public override void OnSecuritiesChanged(SecurityChanges changes)
{
_changes = changes;
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="security">The security to remove subscriptions for</param>
public void RemoveSubscription(Security security)
{
// remove the subscription from our collection
Subscription subscription;
if (_subscriptions.TryRemove(new SymbolSecurityType(security), out subscription))
{
subscription.Dispose();
}
_exchange.RemoveHandler(security.Symbol);
// request to unsubscribe from the subscription
if (!security.SubscriptionDataConfig.IsCustomData)
{
_dataQueueHandler.Unsubscribe(_job, new Dictionary<SecurityType, List<string>>
{
{security.Type, new List<string> {security.Symbol}}
});
}
// keep track of security changes, we emit these to the algorithm
// as notications, used in universe selection
_changes += SecurityChanges.Removed(security);
}
/// <summary>
/// Event invocator for the <see cref="UniverseSelection"/> event
/// </summary>
/// <param name="universe">The universe to perform selection on the data</param>
/// <param name="config">The configuration of the universe</param>
/// <param name="dateTimeUtc">The current date time in UTC</param>
/// <param name="data">The universe selection data to be operated on</param>
protected virtual void OnUniverseSelection(Universe universe, SubscriptionDataConfig config, DateTime dateTimeUtc, IReadOnlyList<BaseData> data)
{
if (UniverseSelection != null)
{
var eventArgs = new UniverseSelectionEventArgs(universe, config, dateTimeUtc, data);
var multicast = (MulticastDelegate)UniverseSelection;
foreach (UniverseSelectionHandler handler in multicast.GetInvocationList())
{
_changes += handler(this, eventArgs);
}
}
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="subscription">The subscription to be removed</param>
/// <returns>True if the subscription was successfully removed, false otherwise</returns>
public bool RemoveSubscription(Subscription subscription)
{
var security = subscription.Security;
// remove the subscriptions
if (subscription.Configuration.IsCustomData)
{
_customExchange.RemoveEnumerator(security.Symbol);
_customExchange.RemoveDataHandler(security.Symbol);
}
else
{
_dataQueueHandler.Unsubscribe(_job, new[] {security.Symbol});
_exchange.RemoveDataHandler(security.Symbol);
}
// remove the subscription from our collection
Subscription sub;
if (_subscriptions.TryRemove(subscription.Security.Symbol, out sub))
{
sub.Dispose();
}
else
{
return false;
}
Log.Trace("LiveTradingDataFeed.RemoveSubscription(): Removed " + security.Symbol.ToString());
// keep track of security changes, we emit these to the algorithm
// as notications, used in universe selection
_changes += SecurityChanges.Removed(security);
// update our fill forward resolution setting
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return true;
}
/// <summary>
/// Main routine for datafeed analysis.
/// </summary>
/// <remarks>This is a hot-thread and should be kept extremely lean. Modify with caution.</remarks>
public void Run()
{
var frontier = DateTime.MaxValue;
try
{
// compute initial frontier time
frontier = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.Run(): Begin: {0} UTC", frontier));
// continue to loop over each subscription, enqueuing data in time order
while (!_cancellationTokenSource.IsCancellationRequested)
{
// each time step reset our security changes
_changes = SecurityChanges.None;
var earlyBirdTicks = long.MaxValue;
var data = new List<KeyValuePair<Security, List<BaseData>>>();
// we union subscriptions with itself so if subscriptions changes on the first
// iteration we will pick up those changes in the union call, this is used in
// universe selection. an alternative is to extract this into a method and check
// to see if changes != SecurityChanges.None, and re-run all subscriptions again,
// This was added as quick fix due to an issue found in universe selection regression alg
foreach (var subscription in Subscriptions.Union(Subscriptions))
{
if (subscription.EndOfStream)
{
// remove finished subscriptions
Subscription sub;
_subscriptions.TryRemove(subscription.Security.Symbol, out sub);
continue;
}
var cache = new KeyValuePair<Security, List<BaseData>>(subscription.Security, new List<BaseData>());
data.Add(cache);
var configuration = subscription.Configuration;
var offsetProvider = subscription.OffsetProvider;
var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier);
while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks)
{
// we want bars rounded using their subscription times, we make a clone
// so we don't interfere with the enumerator's internal logic
var clone = subscription.Current.Clone(subscription.Current.IsFillForward);
clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours);
cache.Value.Add(clone);
if (!subscription.MoveNext())
{
Log.Trace("FileSystemDataFeed.Run(): Finished subscription: " + subscription.Security.Symbol.ToString() + " at " + frontier + " UTC");
break;
}
}
// we have new universe data to select based on
if (subscription.IsUniverseSelectionSubscription && cache.Value.Count > 0)
{
var universe = subscription.Universe;
// always wait for other thread
if (!Bridge.Wait(Timeout.Infinite, _cancellationTokenSource.Token))
{
break;
}
OnUniverseSelection(universe, frontier, configuration, cache.Value);
}
if (subscription.Current != null)
{
// take the earliest between the next piece of data or the next tz discontinuity
earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity()));
}
}
if (earlyBirdTicks == long.MaxValue)
{
if (_changes == SecurityChanges.None)
{
// there's no more data to pull off, we're done
break;
}
}
// enqueue our next time slice and set the frontier for the next
Bridge.Add(TimeSlice.Create(frontier, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes), _cancellationTokenSource.Token);
// never go backwards in time, so take the max between early birds and the current frontier
frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
}
if (!_cancellationTokenSource.IsCancellationRequested)
{
Bridge.CompleteAdding();
}
}
catch (Exception err)
{
Log.Error("FileSystemDataFeed.Run(): Encountered an error: " + err.Message);
if (!_cancellationTokenSource.IsCancellationRequested)
{
Bridge.CompleteAdding();
_cancellationTokenSource.Cancel();
}
}
finally
{
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", frontier));
//Close up all streams:
foreach (var subscription in Subscriptions)
{
subscription.Dispose();
}
Log.Trace("FileSystemDataFeed.Run(): Ending Thread... ");
IsActive = false;
}
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="security">The security to remove subscriptions for</param>
public void RemoveSubscription(Security security)
{
Subscription subscription;
if (!_subscriptions.TryRemove(new SymbolSecurityType(security), out subscription))
{
Log.Error("FileSystemDataFeed.RemoveSubscription(): Unable to remove: " + security.Symbol);
}
_changes += new SecurityChanges(new List<Security>(), new List<Security> {security});
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="universe">The universe the subscription is to be added to</param>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <returns>True if the subscription was created and added successfully, false otherwise</returns>
public bool AddSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
// create and add the subscription to our collection
var subscription = CreateSubscription(universe, security, utcStartTime, utcEndTime);
// for some reason we couldn't create the subscription
if (subscription == null)
{
Log.Trace("Unable to add subscription for: " + security.Symbol.ToString());
return false;
}
Log.Trace("LiveTradingDataFeed.AddSubscription(): Added " + security.Symbol.ToString());
_subscriptions[subscription.Security.Symbol] = subscription;
// send the subscription for the new symbol through to the data queuehandler
// unless it is custom data, custom data is retrieved using the same as backtest
if (!subscription.Configuration.IsCustomData)
{
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>>
{
{security.Type, new List<Symbol> {security.Symbol}}
});
}
// keep track of security changes, we emit these to the algorithm
// as notifications, used in universe selection
_changes += SecurityChanges.Added(security);
// update our fill forward resolution setting
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return true;
}
/// <summary>
/// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
/// </summary>
/// <param name="utcDateTime">The UTC frontier date time</param>
/// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
/// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
/// <param name="data">The data in this <see cref="TimeSlice"/></param>
/// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
/// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<DataFeedPacket> data, SecurityChanges changes)
{
int count = 0;
var security = new List<UpdateData<Security>>();
var custom = new List<UpdateData<Security>>();
var consolidator = new List<UpdateData<SubscriptionDataConfig>>();
var allDataForAlgorithm = new List<BaseData>(data.Count);
var cash = new List<UpdateData<Cash>>(cashBook.Count);
var cashSecurities = new HashSet<Symbol>();
foreach (var cashItem in cashBook.Values)
{
cashSecurities.Add(cashItem.SecuritySymbol);
}
Split split;
Dividend dividend;
Delisting delisting;
SymbolChangedEvent symbolChange;
// we need to be able to reference the slice being created in order to define the
// evaluation of option price models, so we define a 'future' that can be referenced
// in the option price model evaluation delegates for each contract
Slice slice = null;
var sliceFuture = new Lazy<Slice>(() => slice);
var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
var tradeBars = new TradeBars(algorithmTime);
var quoteBars = new QuoteBars(algorithmTime);
var ticks = new Ticks(algorithmTime);
var splits = new Splits(algorithmTime);
var dividends = new Dividends(algorithmTime);
var delistings = new Delistings(algorithmTime);
var optionChains = new OptionChains(algorithmTime);
var symbolChanges = new SymbolChangedEvents(algorithmTime);
foreach (var packet in data)
{
var list = packet.Data;
var symbol = packet.Security.Symbol;
if (list.Count == 0) continue;
// keep count of all data points
if (list.Count == 1 && list[0] is BaseDataCollection)
{
var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count;
if (baseDataCollectionCount == 0)
{
continue;
}
count += baseDataCollectionCount;
}
else
{
count += list.Count;
}
if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData)
{
// This is all the custom data
custom.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, list));
}
var securityUpdate = new List<BaseData>(list.Count);
var consolidatorUpdate = new List<BaseData>(list.Count);
for (int i = 0; i < list.Count; i++)
{
var baseData = list[i];
if (!packet.Configuration.IsInternalFeed)
{
// this is all the data that goes into the algorithm
allDataForAlgorithm.Add(baseData);
}
// don't add internal feed data to ticks/bars objects
if (baseData.DataType != MarketDataType.Auxiliary)
{
if (!packet.Configuration.IsInternalFeed)
{
PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains);
// special handling of options data to build the option chain
if (packet.Security.Type == SecurityType.Option)
{
if (baseData.DataType == MarketDataType.OptionChain)
{
optionChains[baseData.Symbol] = (OptionChain) baseData;
}
else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture))
{
continue;
}
}
// this is data used to update consolidators
consolidatorUpdate.Add(baseData);
}
// this is the data used set market prices
securityUpdate.Add(baseData);
}
// include checks for various aux types so we don't have to construct the dictionaries in Slice
else if ((delisting = baseData as Delisting) != null)
{
delistings[symbol] = delisting;
}
else if ((dividend = baseData as Dividend) != null)
{
dividends[symbol] = dividend;
}
else if ((split = baseData as Split) != null)
{
splits[symbol] = split;
}
else if ((symbolChange = baseData as SymbolChangedEvent) != null)
{
// symbol changes is keyed by the requested symbol
symbolChanges[packet.Configuration.Symbol] = symbolChange;
}
}
if (securityUpdate.Count > 0)
{
// check for 'cash securities' if we found valid update data for this symbol
// and we need this data to update cash conversion rates, long term we should
// have Cash hold onto it's security, then he can update himself, or rather, just
// patch through calls to conversion rate to compue it on the fly using Security.Price
if (cashSecurities.Contains(packet.Security.Symbol))
{
foreach (var cashKvp in cashBook)
{
if (cashKvp.Value.SecuritySymbol == packet.Security.Symbol)
{
var cashUpdates = new List<BaseData> {securityUpdate[securityUpdate.Count - 1]};
cash.Add(new UpdateData<Cash>(cashKvp.Value, packet.Configuration.Type, cashUpdates));
}
}
}
security.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, securityUpdate));
}
if (consolidatorUpdate.Count > 0)
{
consolidator.Add(new UpdateData<SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate));
}
}
slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);
return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="subscription">The subscription to be removed</param>
/// <returns>True if the subscription was successfully removed, false otherwise</returns>
public bool RemoveSubscription(Subscription subscription)
{
var security = subscription.Security;
_exchange.RemoveHandler(security.Symbol);
// request to unsubscribe from the subscription
if (!security.SubscriptionDataConfig.IsCustomData)
{
_dataQueueHandler.Unsubscribe(_job, new Dictionary<SecurityType, List<Symbol>>
{
{security.Type, new List<Symbol> {security.Symbol}}
});
}
// remove the subscription from our collection
Subscription sub;
if (_subscriptions.TryRemove(subscription.Security.Symbol, out sub))
{
sub.Dispose();
}
else
{
return false;
}
Log.Trace("LiveTradingDataFeed.RemoveSubscription(): Removed " + security.Symbol.ToString());
// keep track of security changes, we emit these to the algorithm
// as notications, used in universe selection
_changes += SecurityChanges.Removed(security);
// update our fill forward resolution setting
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return true;
}
/// <summary>
/// Initializes a new <see cref="TimeSlice"/> containing the specified data
/// </summary>
public TimeSlice(DateTime time,
int dataPointCount,
Slice slice,
List<DataFeedPacket> data,
List<UpdateData<Cash>> cashBookUpdateData,
List<UpdateData<Security>> securitiesUpdateData,
List<UpdateData<SubscriptionDataConfig>> consolidatorUpdateData,
List<UpdateData<Security>> customData,
SecurityChanges securityChanges)
{
Time = time;
Data = data;
Slice = slice;
CustomData = customData;
DataPointCount = dataPointCount;
CashBookUpdateData = cashBookUpdateData;
SecuritiesUpdateData = securitiesUpdateData;
ConsolidatorUpdateData = consolidatorUpdateData;
SecurityChanges = securityChanges;
}
/// <summary>
/// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
/// </summary>
/// <param name="utcDateTime">The UTC frontier date time</param>
/// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
/// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
/// <param name="data">The data in this <see cref="TimeSlice"/></param>
/// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
/// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<KeyValuePair<Security, List<BaseData>>> data, SecurityChanges changes)
{
int count = 0;
var security = new List<KeyValuePair<Security, BaseData>>();
var custom = new List<KeyValuePair<Security, List<BaseData>>>();
var consolidator = new List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>>();
var allDataForAlgorithm = new List<BaseData>(data.Count);
var cash = new List<KeyValuePair<Cash, BaseData>>(cashBook.Count);
var cashSecurities = new HashSet<Symbol>();
foreach (var cashItem in cashBook.Values)
{
cashSecurities.Add(cashItem.SecuritySymbol);
}
Split split;
Dividend dividend;
Delisting delisting;
SymbolChangedEvent symbolChange;
var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
var tradeBars = new TradeBars(algorithmTime);
var ticks = new Ticks(algorithmTime);
var splits = new Splits(algorithmTime);
var dividends = new Dividends(algorithmTime);
var delistings = new Delistings(algorithmTime);
var symbolChanges = new SymbolChangedEvents(algorithmTime);
foreach (var kvp in data)
{
var list = kvp.Value;
var symbol = kvp.Key.Symbol;
// keep count of all data points
if (list.Count == 1 && list[0] is BaseDataCollection)
{
count += ((BaseDataCollection) list[0]).Data.Count;
}
else
{
count += list.Count;
}
BaseData update = null;
var consolidatorUpdate = new List<BaseData>(list.Count);
for (int i = 0; i < list.Count; i++)
{
var baseData = list[i];
if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
{
// this is all the data that goes into the algorithm
allDataForAlgorithm.Add(baseData);
if (kvp.Key.SubscriptionDataConfig.IsCustomData)
{
// this is all the custom data
custom.Add(kvp);
}
}
// don't add internal feed data to ticks/bars objects
if (baseData.DataType != MarketDataType.Auxiliary)
{
if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
{
// populate ticks and tradebars dictionaries with no aux data
if (baseData.DataType == MarketDataType.Tick)
{
List<Tick> ticksList;
if (!ticks.TryGetValue(symbol, out ticksList))
{
ticksList = new List<Tick> {(Tick) baseData};
ticks[symbol] = ticksList;
}
ticksList.Add((Tick) baseData);
}
else if (baseData.DataType == MarketDataType.TradeBar)
{
tradeBars[symbol] = (TradeBar) baseData;
}
// this is data used to update consolidators
consolidatorUpdate.Add(baseData);
}
// this is the data used set market prices
update = baseData;
}
// include checks for various aux types so we don't have to construct the dictionaries in Slice
else if ((delisting = baseData as Delisting) != null)
{
delistings[symbol] = delisting;
}
else if ((dividend = baseData as Dividend) != null)
{
dividends[symbol] = dividend;
}
else if ((split = baseData as Split) != null)
{
splits[symbol] = split;
}
else if ((symbolChange = baseData as SymbolChangedEvent) != null)
{
// symbol changes is keyed by the requested symbol
symbolChanges[kvp.Key.SubscriptionDataConfig.Symbol] = symbolChange;
}
}
// check for 'cash securities' if we found valid update data for this symbol
// and we need this data to update cash conversion rates, long term we should
// have Cash hold onto it's security, then he can update himself, or rather, just
// patch through calls to conversion rate to compue it on the fly using Security.Price
if (update != null && cashSecurities.Contains(kvp.Key.Symbol))
{
foreach (var cashKvp in cashBook)
{
if (cashKvp.Value.SecuritySymbol == kvp.Key.Symbol)
{
cash.Add(new KeyValuePair<Cash, BaseData>(cashKvp.Value, update));
}
}
}
security.Add(new KeyValuePair<Security, BaseData>(kvp.Key, update));
consolidator.Add(new KeyValuePair<SubscriptionDataConfig, List<BaseData>>(kvp.Key.SubscriptionDataConfig, consolidatorUpdate));
}
var slice = new Slice(utcDateTime.ConvertFromUtc(algorithmTimeZone), allDataForAlgorithm, tradeBars, ticks, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);
return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
}
/// <summary>
/// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
/// </summary>
/// <param name="data">Slice object keyed by symbol containing the stock data</param>
public override void OnData(Slice data)
{
if (Transactions.OrdersCount == 0)
{
MarketOrder("SPY", 100);
}
foreach (var kvp in data.Delistings)
{
_delistedSymbols.Add(kvp.Key);
}
if (_changes != null && _changes.AddedSecurities.All(x => data.Bars.ContainsKey(x.Symbol)))
{
foreach (var security in _changes.AddedSecurities)
{
Log(Time + ": Added Security: " + security.Symbol.ID);
MarketOnOpenOrder(security.Symbol, 100);
}
foreach (var security in _changes.RemovedSecurities)
{
Log(Time + ": Removed Security: " + security.Symbol.ID);
if (!_delistedSymbols.Contains(security.Symbol))
{
MarketOnOpenOrder(security.Symbol, -100);
}
}
_changes = null;
}
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <param name="isUserDefinedSubscription">Set to true to prevent coarse universe selection from removing this subscription</param>
public void AddSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
// create and add the subscription to our collection
var subscription = CreateSubscription(security, utcStartTime, utcEndTime, isUserDefinedSubscription);
// for some reason we couldn't create the subscription
if (subscription == null)
{
Log.Trace("Unable to add subscription for: " + security.Symbol);
return;
}
_subscriptions[new SymbolSecurityType(subscription)] = subscription;
// send the subscription for the new symbol through to the data queuehandler
// unless it is custom data, custom data is retrieved using the same as backtest
if (!subscription.Configuration.IsCustomData)
{
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>>
{
{security.Type, new List<string> {security.Symbol}}
});
}
// keep track of security changes, we emit these to the algorithm
// as notifications, used in universe selection
_changes += SecurityChanges.Added(security);
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="universe">The universe the subscription is to be added to</param>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
public bool AddSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
var subscription = CreateSubscription(universe, _resultHandler, security, utcStartTime, utcEndTime, _fillForwardResolution);
if (subscription == null)
{
// subscription will be null when there's no tradeable dates for the security between the requested times, so
// don't even try to load the data
return false;
}
Log.Debug("FileSystemDataFeed.AddSubscription(): Added " + security.Symbol.ID + " Start: " + utcStartTime + " End: " + utcEndTime);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
// prime the pump, run method checks current before move next calls
//PrimeSubscriptionPump(subscription, true);
_changes += SecurityChanges.Added(security);
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return true;
}
/// <summary>
/// Primary entry point.
/// </summary>
public void Run()
{
IsActive = true;
// we want to emit to the bridge minimally once a second since the data feed is
// the heartbeat of the application, so this value will contain a second after
// the last emit time, and if we pass this time, we'll emit even with no data
var nextEmit = DateTime.MinValue;
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
// perform sleeps to wake up on the second?
var frontier = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTime(frontier);
var data = new List<KeyValuePair<Security, List<BaseData>>>();
foreach (var kvp in _subscriptions)
{
var subscription = kvp.Value;
var cache = new KeyValuePair<Security, List<BaseData>>(subscription.Security, new List<BaseData>());
// dequeue data that is time stamped at or before this frontier
while (subscription.MoveNext() && subscription.Current != null)
{
cache.Value.Add(subscription.Current);
}
// if we have data, add it to be added to the bridge
if (cache.Value.Count > 0) data.Add(cache);
// we have new universe data to select based on
if (subscription.IsUniverseSelectionSubscription && cache.Value.Count > 0)
{
var universe = subscription.Universe;
// always wait for other thread to sync up
if (!Bridge.Wait(Timeout.Infinite, _cancellationTokenSource.Token))
{
break;
}
// fire the universe selection event
OnUniverseSelection(universe, subscription.Configuration, frontier, cache.Value);
}
}
// check for cancellation
if (_cancellationTokenSource.IsCancellationRequested) return;
// emit on data or if we've elapsed a full second since last emit
if (data.Count != 0 || frontier >= nextEmit)
{
Bridge.Add(TimeSlice.Create(frontier, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes));
// force emitting every second
nextEmit = frontier.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
// reset our security changes
_changes = SecurityChanges.None;
// take a short nap
Thread.Sleep(1);
}
}
catch (Exception err)
{
Log.Error(err);
}
IsActive = false;
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="subscription">The subscription to be removed</param>
public bool RemoveSubscription(Subscription subscription)
{
Subscription sub;
if (!_subscriptions.TryRemove(subscription.Security.Symbol, out sub))
{
Log.Error("FileSystemDataFeed.RemoveSubscription(): Unable to remove: " + subscription.Security.Symbol.ToString());
return false;
}
Log.Debug("FileSystemDataFeed.RemoveSubscription(): Removed " + subscription.Security.Symbol.ToString());
_changes += SecurityChanges.Removed(sub.Security);
_fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm);
return true;
}
/// <summary>
/// Event fired each time the we add/remove securities from the data feed
/// </summary>
/// <param name="changes"></param>
public override void OnSecuritiesChanged(SecurityChanges changes)
{
// each time our securities change we'll be notified here
_changes = changes;
}
/// <summary>
/// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
/// </summary>
/// <param name="data">Slice object keyed by symbol containing the stock data</param>
public override void OnData(Slice data)
{
if (Transactions.OrdersCount == 0)
{
MarketOrder("SPY", 100);
}
foreach (var kvp in data.Delistings)
{
_delistedSymbols.Add(kvp.Key);
}
if (Time.Date == new DateTime(2014, 04, 07))
{
Liquidate();
return;
}
if (_changes != null && _changes.AddedSecurities.All(x => data.Bars.ContainsKey(x.Symbol)))
{
foreach (var security in _changes.AddedSecurities)
{
Console.WriteLine(Time + ": Added Security: " + security.Symbol);
MarketOnOpenOrder(security.Symbol, 100);
}
foreach (var security in _changes.RemovedSecurities)
{
Console.WriteLine(Time + ": Removed Security: " + security.Symbol);
if (!_delistedSymbols.Contains(security.Symbol))
{
MarketOnOpenOrder(security.Symbol, -100);
}
}
_changes = null;
}
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param>
/// <returns>True if the subscription was created and added successfully, false otherwise</returns>
public bool AddSubscription(SubscriptionRequest request)
{
if (_subscriptions.Contains(request.Configuration))
{
// duplicate subscription request
return false;
}
// create and add the subscription to our collection
var subscription = request.IsUniverseSubscription
? CreateUniverseSubscription(request)
: CreateSubscription(request);
// for some reason we couldn't create the subscription
if (subscription == null)
{
Log.Trace("Unable to add subscription for: " + request.Configuration);
return false;
}
Log.Trace("LiveTradingDataFeed.AddSubscription(): Added " + request.Configuration);
_subscriptions.TryAdd(subscription);
// send the subscription for the new symbol through to the data queuehandler
// unless it is custom data, custom data is retrieved using the same as backtest
if (!subscription.Configuration.IsCustomData)
{
_dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});
}
// keep track of security changes, we emit these to the algorithm
// as notifications, used in universe selection
_changes += SecurityChanges.Added(request.Security);
UpdateFillForwardResolution();
return true;
}
public override void OnSecuritiesChanged(SecurityChanges changes)
{
if (changes.AddedSecurities.Count != 0)
{
Log("Security added: " + string.Join(",", changes.AddedSecurities.Select(x => x.Symbol)));
foreach (var security in changes.AddedSecurities.OrderBy(x => x.Symbol))
{
if (!security.HoldStock)
{
SetHoldings(security.Symbol, 0.0075);
}
}
}
if (changes.RemovedSecurities.Count != 0)
{
Log("Security removed: " + string.Join(",", changes.RemovedSecurities.Select(x => x.Symbol)));
foreach (var security in changes.RemovedSecurities.OrderBy(x => x.Symbol))
{
var previousOrders = Transactions.GetOrders(x => x.Symbol == security.Symbol).OrderByDescending(x => x.Time);
if (security.HoldStock && previousOrders.First().Time + TimeSpan.FromDays(30) < Time)
{
Log("Liquidating: " + security.Symbol);
Liquidate(security.Symbol);
}
}
}
}
/// <summary>
/// Primary entry point.
/// </summary>
public void Run()
{
IsActive = true;
// we want to emit to the bridge minimally once a second since the data feed is
// the heartbeat of the application, so this value will contain a second after
// the last emit time, and if we pass this time, we'll emit even with no data
var nextEmit = DateTime.MinValue;
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
// perform sleeps to wake up on the second?
_frontierUtc = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTime(_frontierUtc);
var data = new List<DataFeedPacket>();
foreach (var subscription in Subscriptions)
{
var packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
// dequeue data that is time stamped at or before this frontier
while (subscription.MoveNext() && subscription.Current != null)
{
packet.Add(subscription.Current);
}
// if we have data, add it to be added to the bridge
if (packet.Count > 0) data.Add(packet);
// we have new universe data to select based on
if (subscription.IsUniverseSelectionSubscription && packet.Count > 0)
{
var universe = subscription.Universe;
// always wait for other thread to sync up
if (!_bridge.WaitHandle.WaitOne(Timeout.Infinite, _cancellationTokenSource.Token))
{
break;
}
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var collection = packet.Data[0] as BaseDataCollection ?? new BaseDataCollection(_frontierUtc, subscription.Configuration.Symbol, packet.Data);
_changes += _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
}
}
// check for cancellation
if (_cancellationTokenSource.IsCancellationRequested) return;
// emit on data or if we've elapsed a full second since last emit
if (data.Count != 0 || _frontierUtc >= nextEmit)
{
_bridge.Add(TimeSlice.Create(_frontierUtc, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes), _cancellationTokenSource.Token);
// force emitting every second
nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
// reset our security changes
_changes = SecurityChanges.None;
// take a short nap
Thread.Sleep(1);
}
}
catch (Exception err)
{
Log.Error(err);
_algorithm.RunTimeError = err;
}
Log.Trace("LiveTradingDataFeed.Run(): Exited thread.");
IsActive = false;
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="symbol">The symbol of the subscription to be removed</param>
/// <returns>True if the subscription was successfully removed, false otherwise</returns>
public bool RemoveSubscription(Symbol symbol)
{
List<Subscription> subscriptions;
if (!_subscriptions.TryRemove(symbol, out subscriptions))
{
Log.Error("FileSystemDataFeed.RemoveSubscription(): Unable to remove: " + symbol.ToString());
return false;
}
foreach (var subscription in subscriptions)
{
subscription.Dispose();
}
Log.Debug("FileSystemDataFeed.RemoveSubscription(): Removed " + symbol.ToString());
_changes += SecurityChanges.Removed(subscriptions.Select(x => x.Security).ToArray());
UpdateFillForwardResolution();
return true;
}
