/// <summary> /// Syncs the specifies subscriptions at the frontier time /// </summary> /// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier. /// This value will equal DateTime.MaxValue when the subscriptions are all finished</param> /// <returns>A time slice for the specified frontier time</returns> public TimeSlice Sync(DateTime frontier, IEnumerable<Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier) { var changes = SecurityChanges.None; nextFrontier = DateTime.MaxValue; var earlyBirdTicks = nextFrontier.Ticks; var data = new List<DataFeedPacket>(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security); data.Add(packet); var configuration = subscription.Configuration; var offsetProvider = subscription.OffsetProvider; var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours); packet.Add(clone); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } // we have new universe data to select based on if (subscription.IsUniverseSelectionSubscription && packet.Count > 0) { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var collection = packet.Data[0] as BaseDataCollection ?? new BaseDataCollection(frontier, subscription.Configuration.Symbol, packet.Data); newChanges += _universeSelection.ApplyUniverseSelection(subscription.Universe, frontier, collection); } if (subscription.Current != null) { // take the earliest between the next piece of data or the next tz discontinuity earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity())); } } changes += newChanges; } while (newChanges != SecurityChanges.None); nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); return TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes); }
/// <summary> /// Primary entry point. /// </summary> public void Run() { IsActive = true; // we want to emit to the bridge minimally once a second since the data feed is // the heartbeat of the application, so this value will contain a second after // the last emit time, and if we pass this time, we'll emit even with no data var nextEmit = DateTime.MinValue; try { while (!_cancellationTokenSource.IsCancellationRequested) { // perform sleeps to wake up on the second? _frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTime(_frontierUtc); var data = new List<DataFeedPacket>(); foreach (var subscription in Subscriptions) { var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); // dequeue data that is time stamped at or before this frontier while (subscription.MoveNext() && subscription.Current != null) { packet.Add(subscription.Current); } // if we have data, add it to be added to the bridge if (packet.Count > 0) data.Add(packet); // we have new universe data to select based on if (subscription.IsUniverseSelectionSubscription && packet.Count > 0) { var universe = subscription.Universe; // always wait for other thread to sync up if (!_bridge.WaitHandle.WaitOne(Timeout.Infinite, _cancellationTokenSource.Token)) { break; } // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var collection = packet.Data[0] as BaseDataCollection ?? new BaseDataCollection(_frontierUtc, subscription.Configuration.Symbol, packet.Data); _changes += _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); } } // check for cancellation if (_cancellationTokenSource.IsCancellationRequested) return; // emit on data or if we've elapsed a full second since last emit if (data.Count != 0 || _frontierUtc >= nextEmit) { _bridge.Add(TimeSlice.Create(_frontierUtc, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes), _cancellationTokenSource.Token); // force emitting every second nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } // reset our security changes _changes = SecurityChanges.None; // take a short nap Thread.Sleep(1); } } catch (Exception err) { Log.Error(err); _algorithm.RunTimeError = err; } Log.Trace("LiveTradingDataFeed.Run(): Exited thread."); IsActive = false; }
/// <summary> /// Primary entry point. /// </summary> public void Run() { IsActive = true; // we want to emit to the bridge minimally once a second since the data feed is // the heartbeat of the application, so this value will contain a second after // the last emit time, and if we pass this time, we'll emit even with no data var nextEmit = DateTime.MinValue; try { while (!_cancellationTokenSource.IsCancellationRequested) { // perform sleeps to wake up on the second? _frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTime(_frontierUtc); var data = new List <DataFeedPacket>(); foreach (var subscription in Subscriptions) { var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); // dequeue data that is time stamped at or before this frontier while (subscription.MoveNext() && subscription.Current != null) { packet.Add(subscription.Current); } // if we have data, add it to be added to the bridge if (packet.Count > 0) { data.Add(packet); } // we have new universe data to select based on if (subscription.IsUniverseSelectionSubscription && packet.Count > 0) { var universe = subscription.Universe; // always wait for other thread to sync up if (!_bridge.WaitHandle.WaitOne(Timeout.Infinite, _cancellationTokenSource.Token)) { break; } // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var collection = packet.Data[0] as BaseDataCollection ?? new BaseDataCollection(_frontierUtc, subscription.Configuration.Symbol, packet.Data); _changes += _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); } } // check for cancellation if (_cancellationTokenSource.IsCancellationRequested) { return; } // emit on data or if we've elapsed a full second since last emit if (data.Count != 0 || _frontierUtc >= nextEmit) { _bridge.Add(TimeSlice.Create(_frontierUtc, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, data, _changes), _cancellationTokenSource.Token); // force emitting every second nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } // reset our security changes _changes = SecurityChanges.None; // take a short nap Thread.Sleep(1); } } catch (Exception err) { Log.Error(err); _algorithm.RunTimeError = err; _algorithm.Status = AlgorithmStatus.RuntimeError; // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); if (!_cancellationTokenSource.IsCancellationRequested) { _bridge.Add( TimeSlice.Create(nextEmit, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, new List <DataFeedPacket>(), SecurityChanges.None), _cancellationTokenSource.Token); } } Log.Trace("LiveTradingDataFeed.Run(): Exited thread."); IsActive = false; }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { var delayedSubscriptionFinished = false; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse); while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished = true; break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested || delayedSubscriptionFinished) { delayedSubscriptionFinished = false; // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = TimeSlice.Create(frontierUtc, _sliceTimeZone, _cashBook, data, changes, universeDataForTimeSliceCreate); return(timeSlice); }
/// <summary> /// Syncs the specifies subscriptions at the frontier time /// </summary> /// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier. /// This value will equal DateTime.MaxValue when the subscriptions are all finished</param> /// <returns>A time slice for the specified frontier time</returns> public TimeSlice Sync(DateTime frontier, IEnumerable <Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier) { var changes = SecurityChanges.None; nextFrontier = DateTime.MaxValue; var earlyBirdTicks = nextFrontier.Ticks; var data = new List <DataFeedPacket>(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); data.Add(packet); var configuration = subscription.Configuration; var offsetProvider = subscription.OffsetProvider; var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours); packet.Add(clone); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } // we have new universe data to select based on if (subscription.IsUniverseSelectionSubscription && packet.Count > 0) { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var collection = packet.Data[0] as BaseDataCollection ?? new BaseDataCollection(frontier, subscription.Configuration.Symbol, packet.Data); newChanges += _universeSelection.ApplyUniverseSelection(subscription.Universe, frontier, collection); } if (subscription.Current != null) { // take the earliest between the next piece of data or the next tz discontinuity earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity())); } } changes += newChanges; }while (newChanges != SecurityChanges.None); nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); return(TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes)); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { long earlyBirdTicks; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); SecurityChanges newChanges; do { earlyBirdTicks = MaxDateTimeTicks; newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } if (subscription.Current == null) { throw new Exception($"SubscriptionSynchronizer.Sync(): subscription.Current is null, configuration: {subscription}"); } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse); while (subscription.Current.EmitTimeUtc <= _frontier) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universe, out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universe] = collection; } // remove subscription for universe data if disposal requested AFTER time sync // this ensures we get any security changes from removing the universe and its children if (subscription.Universe.DisposeRequested) { OnSubscriptionFinished(subscription); } } } if (subscription.Current != null) { if (earlyBirdTicks == MaxDateTimeTicks) { earlyBirdTicks = subscription.Current.EmitTimeUtc.Ticks; } else { // take the earliest between the next piece of data or the current earliest bird earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks); } } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, _frontier, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = TimeSlice.Create(_frontier, _sliceTimeZone, _cashBook, data, changes, universeDataForTimeSliceCreate); // next frontier time _frontier = new DateTime(Math.Max(earlyBirdTicks, _frontier.Ticks), DateTimeKind.Utc); return(timeSlice); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { long earlyBirdTicks; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); var universeData = new Dictionary <Universe, BaseDataCollection>(); SecurityChanges newChanges; do { earlyBirdTicks = MaxDateTimeTicks; universeData.Clear(); newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); var configuration = subscription.Configuration; var offsetProvider = subscription.OffsetProvider; var currentOffsetTicks = offsetProvider.GetOffsetTicks(_frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= _frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.ExchangeRoundDownInTimeZone(configuration.Increment, subscription.Security.Exchange.Hours, configuration.DataTimeZone, configuration.ExtendedMarketHours); packet.Add(clone); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universe, out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = subscription.Current as OptionChainUniverseDataCollection; var underlying = current != null ? current.Underlying : null; collection = new OptionChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData, underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { var current = subscription.Current as FuturesChainUniverseDataCollection; collection = new FuturesChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universe] = collection; } } } if (subscription.Current != null) { if (earlyBirdTicks == MaxDateTimeTicks) { earlyBirdTicks = subscription.Current.EndTime.ConvertToUtc(subscription.TimeZone).Ticks; } else { // take the earliest between the next piece of data or the next tz discontinuity earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity())); } } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; newChanges += _universeSelection.ApplyUniverseSelection(universe, _frontier, baseDataCollection); } changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = TimeSlice.Create(_frontier, _sliceTimeZone, _cashBook, data, changes); // next frontier time _frontier = new DateTime(Math.Max(earlyBirdTicks, _frontier.Ticks), DateTimeKind.Utc); return(timeSlice); }