/// <summary>
/// Initializes a new <see cref="TimeSlice"/> containing the specified data
/// </summary>
public TimeSlice(DateTime time, int dataPointCount, Slice slice, List<KeyValuePair<Security, List<BaseData>>> data, List<KeyValuePair<Cash, BaseData>> cashBookUpdateData, List<KeyValuePair<Security, BaseData>> securitiesUpdateData, List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>> consolidatorUpdateData, List<KeyValuePair<Security, List<BaseData>>> customData, SecurityChanges securityChanges)
{
Time = time;
Data = data;
Slice = slice;
CustomData = customData;
DataPointCount = dataPointCount;
CashBookUpdateData = cashBookUpdateData;
SecuritiesUpdateData = securitiesUpdateData;
ConsolidatorUpdateData = consolidatorUpdateData;
SecurityChanges = securityChanges;
}
public override void OnSecuritiesChanged(SecurityChanges changes)
{
if (changes.AddedSecurities.Count != 0)
{
Log("Security added: " + string.Join(",", changes.AddedSecurities.Select(x => x.Symbol)));
foreach (var security in changes.AddedSecurities.OrderBy(x => x.Symbol))
{
if (!security.HoldStock)
{
SetHoldings(security.Symbol, 0.0075);
}
}
}
if (changes.RemovedSecurities.Count != 0)
{
Log("Security removed: " + string.Join(",", changes.RemovedSecurities.Select(x => x.Symbol)));
foreach (var security in changes.RemovedSecurities.OrderBy(x => x.Symbol))
{
var previousOrders = Transactions.GetOrders(x => x.Symbol == security.Symbol).OrderByDescending(x => x.Time);
if (security.HoldStock && previousOrders.First().Time + TimeSpan.FromDays(30) < Time)
{
Log("Liquidating: " + security.Symbol);
Liquidate(security.Symbol);
}
}
}
}
/// <summary>
/// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
/// </summary>
/// <param name="algorithm">The algorithm we're creating <see cref="TimeSlice"/> instances for</param>
/// <param name="utcDateTime">The UTC frontier date time</param>
/// <param name="data">The data in this <see cref="TimeSlice"/></param>
/// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
/// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
public static TimeSlice Create(IAlgorithm algorithm, DateTime utcDateTime, List<KeyValuePair<Security, List<BaseData>>> data, SecurityChanges changes)
{
int count = 0;
var security = new List<KeyValuePair<Security, BaseData>>();
var custom = new List<KeyValuePair<Security, List<BaseData>>>();
var consolidator = new List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>>();
var allDataForAlgorithm = new List<BaseData>(data.Count);
var cash = new List<KeyValuePair<Cash, BaseData>>(algorithm.Portfolio.CashBook.Count);
var cashSecurities = new HashSet<Symbol>();
foreach (var cashItem in algorithm.Portfolio.CashBook.Values)
{
cashSecurities.Add(cashItem.SecuritySymbol);
}
Split split;
Dividend dividend;
Delisting delisting;
SymbolChangedEvent symbolChange;
var algorithmTime = utcDateTime.ConvertFromUtc(algorithm.TimeZone);
var tradeBars = new TradeBars(algorithmTime);
var ticks = new Ticks(algorithmTime);
var splits = new Splits(algorithmTime);
var dividends = new Dividends(algorithmTime);
var delistings = new Delistings(algorithmTime);
var symbolChanges = new SymbolChangedEvents(algorithmTime);
foreach (var kvp in data)
{
var list = kvp.Value;
var symbol = kvp.Key.Symbol;
// keep count of all data points
count += list.Count;
BaseData update = null;
var consolidatorUpdate = new List<BaseData>(list.Count);
for (int i = 0; i < list.Count; i++)
{
var baseData = list[i];
if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
{
// this is all the data that goes into the algorithm
allDataForAlgorithm.Add(baseData);
}
if (kvp.Key.IsDynamicallyLoadedData)
{
// this is all the custom data
custom.Add(kvp);
}
if (baseData.DataType != MarketDataType.Auxiliary)
{
// populate ticks and tradebars dictionaries with no aux data
if (baseData.DataType == MarketDataType.Tick)
{
List<Tick> ticksList;
if (!ticks.TryGetValue(symbol, out ticksList))
{
ticksList = new List<Tick> {(Tick) baseData};
ticks[symbol] = ticksList;
}
ticksList.Add((Tick) baseData);
}
else if (baseData.DataType == MarketDataType.TradeBar)
{
tradeBars[symbol] = (TradeBar) baseData;
}
// this is data used to update consolidators
consolidatorUpdate.Add(baseData);
// this is the data used set market prices
update = baseData;
}
// include checks for various aux types so we don't have to construct the dictionaries in Slice
else if ((delisting = baseData as Delisting) != null)
{
delistings[symbol] = delisting;
}
else if ((dividend = baseData as Dividend) != null)
{
dividends[symbol] = dividend;
}
else if ((split = baseData as Split) != null)
{
splits[symbol] = split;
}
else if ((symbolChange = baseData as SymbolChangedEvent) != null)
{
// symbol changes is keyed by the requested symbol
symbolChanges[kvp.Key.SubscriptionDataConfig.Symbol] = symbolChange;
}
}
// check for 'cash securities' if we found valid update data for this symbol
// and we need this data to update cash conversion rates, long term we should
// have Cash hold onto it's security, then he can update himself, or rather, just
// patch through calls to conversion rate to compue it on the fly using Security.Price
if (update != null && cashSecurities.Contains(kvp.Key.Symbol))
{
foreach (var cashKvp in algorithm.Portfolio.CashBook)
{
if (cashKvp.Value.SecuritySymbol == kvp.Key.Symbol)
{
cash.Add(new KeyValuePair<Cash, BaseData>(cashKvp.Value, update));
}
}
}
security.Add(new KeyValuePair<Security, BaseData>(kvp.Key, update));
consolidator.Add(new KeyValuePair<SubscriptionDataConfig, List<BaseData>>(kvp.Key.SubscriptionDataConfig, consolidatorUpdate));
}
var slice = new Slice(utcDateTime.ConvertFromUtc(algorithm.TimeZone), allDataForAlgorithm, tradeBars, ticks, splits, dividends, delistings, symbolChanges);
return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
}
/// <summary>
/// Adds a new subscription to provide data for the specified security.
/// </summary>
/// <param name="security">The security to add a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
public void AddSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime)
{
var subscription = CreateSubscription(_resultHandler, security, utcStartTime, utcEndTime, security.SubscriptionDataConfig.Resolution, false);
if (subscription == null)
{
// subscription will be null when there's no tradeable dates for the security between the requested times, so
// don't even try to load the data
return;
}
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
// prime the pump, run method checks current before move next calls
PrimeSubscriptionPump(subscription, true);
_changes += new SecurityChanges(new List<Security> {security}, new List<Security>());
}
/// <summary>
/// Main routine for datafeed analysis.
/// </summary>
/// <remarks>This is a hot-thread and should be kept extremely lean. Modify with caution.</remarks>
public void Run()
{
var universeSelectionMarkets = new List<string> {"usa"};
var frontier = DateTime.MaxValue;
try
{
// don't initialize universe selection if it's not requested
if (_algorithm.Universe != null)
{
// initialize subscriptions used for universe selection
foreach (var market in universeSelectionMarkets)
{
AddSubscriptionForUniverseSelectionMarket(market);
}
}
// compute initial frontier time
frontier = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.Run(): Begin: {0} UTC", frontier));
// continue to loop over each subscription, enqueuing data in time order
while (!_cancellationTokenSource.IsCancellationRequested)
{
// each time step reset our security changes
_changes = SecurityChanges.None;
var earlyBirdTicks = long.MaxValue;
var data = new List<KeyValuePair<Security, List<BaseData>>>();
// we union subscriptions with itself so if subscriptions changes on the first
// iteration we will pick up those changes in the union call, this is used in
// universe selection. an alternative is to extract this into a method and check
// to see if changes != SecurityChanges.None, and re-run all subscriptions again,
// This was added as quick fix due to an issue found in universe selection regression alg
foreach (var subscription in Subscriptions.Union(Subscriptions))
{
if (subscription.EndOfStream)
{
// skip subscriptions that are finished
continue;
}
var cache = new KeyValuePair<Security, List<BaseData>>(subscription.Security, new List<BaseData>());
data.Add(cache);
var configuration = subscription.Configuration;
var offsetProvider = subscription.OffsetProvider;
var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier);
while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks)
{
// we want bars rounded using their subscription times, we make a clone
// so we don't interfere with the enumerator's internal logic
var clone = subscription.Current.Clone(subscription.Current.IsFillForward);
clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours);
cache.Value.Add(clone);
if (!subscription.MoveNext())
{
Log.Trace("FileSystemDataFeed.Run(): Finished subscription: " + subscription.Security.Symbol + " at " + frontier + " UTC");
break;
}
}
// we have new universe data to select based on
if (subscription.IsFundamentalSubscription && cache.Value.Count > 0)
{
// always wait for other thread
if (!Bridge.Wait(Timeout.Infinite, _cancellationTokenSource.Token))
{
break;
}
OnFundamental(FundamentalType.Coarse, frontier, configuration, cache.Value);
}
if (subscription.Current != null)
{
// take the earliest between the next piece of data or the next tz discontinuity
earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity()));
}
}
if (earlyBirdTicks == long.MaxValue)
{
// there's no more data to pull off, we're done
break;
}
// enqueue our next time slice and set the frontier for the next
Bridge.Add(TimeSlice.Create(frontier, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes), _cancellationTokenSource.Token);
// never go backwards in time, so take the max between early birds and the current frontier
frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
}
if (!_cancellationTokenSource.IsCancellationRequested)
{
Bridge.CompleteAdding();
}
}
catch (Exception err)
{
Log.Error("FileSystemDataFeed.Run(): Encountered an error: " + err.Message);
if (!_cancellationTokenSource.IsCancellationRequested)
{
Bridge.CompleteAdding();
_cancellationTokenSource.Cancel();
}
}
finally
{
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", frontier));
//Close up all streams:
foreach (var subscription in Subscriptions)
{
subscription.Dispose();
}
Log.Trace("FileSystemDataFeed.Run(): Ending Thread... ");
IsActive = false;
}
}
/// <summary>
/// Removes the subscription from the data feed, if it exists
/// </summary>
/// <param name="security">The security to remove subscriptions for</param>
public void RemoveSubscription(Security security)
{
Subscription subscription;
if (!_subscriptions.TryRemove(new SymbolSecurityType(security), out subscription))
{
Log.Error("FileSystemDataFeed.RemoveSubscription(): Unable to remove: " + security.Symbol);
}
_changes += new SecurityChanges(new List<Security>(), new List<Security> {security});
}
/// <summary>
/// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
/// </summary>
/// <param name="data">Slice object keyed by symbol containing the stock data</param>
public override void OnData(Slice data)
{
if (Transactions.OrdersCount == 0)
{
MarketOrder("SPY", 100);
}
foreach (var kvp in data.Delistings)
{
_delistedSymbols.Add(kvp.Key);
}
if (Time.Date == new DateTime(2014, 04, 07))
{
Liquidate();
return;
}
if (_changes != null && _changes.AddedSecurities.All(x => data.Bars.ContainsKey(x.Symbol)))
{
foreach (var security in _changes.AddedSecurities)
{
Console.WriteLine(Time + ": Added Security: " + security.Symbol);
MarketOnOpenOrder(security.Symbol, 100);
}
foreach (var security in _changes.RemovedSecurities)
{
Console.WriteLine(Time + ": Removed Security: " + security.Symbol);
if (!_delistedSymbols.Contains(security.Symbol))
{
MarketOnOpenOrder(security.Symbol, -100);
}
}
_changes = null;
}
}
public override void OnSecuritiesChanged(SecurityChanges changes)
{
_changes = changes;
}
