Example #1
0
        /// <summary>
        /// Execute the primary thread for retrieving stock data.
        /// 1. Subscribe to the streams requested.
        /// 2. Build bars or tick data requested, primary loop increment smallest possible.
        /// </summary>
        public void Run()
        {
            // Symbols requested:
            _symbols = (from security in _algorithm.Securities.Values
                        where !security.IsDynamicallyLoadedData && (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
                        select security.Symbol).ToList<string>();

            //Initialize:
            _streamStore = new Dictionary<int, StreamStore>();

            Log.Trace("LiveTradingDataFeed.Stream(): Initializing subscription stream stores...");
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                var config = _subscriptions[i];
                _streamStore.Add(i, new StreamStore(config));
            }
            Log.Trace("LiveTradingDataFeed.Stream(): Initialized " + _streamStore.Count + " stream stores.");

            // Set up separate thread to handle stream and building packets:
            var streamThread = new Thread(Stream);
            streamThread.Start();
            Thread.Sleep(5); // Wait a little for the other thread to init.

            var storingData = false;

            // Setup Real Time Event Trigger:
            var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), () =>
            {
                storingData = true;

                //This is a minute start / 0-seconds.
                var now = DateTime.Now;
                var onMinute = (now.Second == 0);
                var onDay = ((now.Second == 0) && (now.Hour == 0));

                // Determine if this subscription needs to be archived:
                for (var i = 0; i < Subscriptions.Count; i++)
                {
                    //Do critical events every second regardless of the market/hybernate state:
                    if (onDay)
                    {
                        //Every day refresh the source file for the custom user data:
                        var success = _subscriptionManagers[i].RefreshSource(now.Date);

                        //Update the securities market open/close.
                        UpdateSecurityMarketHours();
                    }

                    //If hybernate stop sending data until its resumed.
                    if (_hibernate) continue;

                    switch (_subscriptions[i].Resolution)
                    {
                        //This is a second resolution data source:
                        case Resolution.Second:
                            //Enqueue our live data:
                            _streamStore[i].TriggerArchive(_subscriptions[i].FillDataForward);
                            Log.Debug("LiveTradingDataFeed.Run(): Triggered Archive: " + _subscriptions[i].Symbol + "-Second... " + now.ToLongTimeString());
                            break;

                        //This is a minute resolution data source:
                        case Resolution.Minute:
                            if (onMinute)
                            {
                                _streamStore[i].TriggerArchive(_subscriptions[i].FillDataForward);
                                Log.Debug("LiveTradingDataFeed.Run(): Triggered Archive: " + _subscriptions[i].Symbol + "-Minute... " + now.ToLongTimeString());
                            }
                            break;
                    }
                }

                storingData = false;
            });

            //Start the realtime sampler above
            realtime.Start();

            // Scan the Stream Stores for Archived Bars,
            do
            {
                while (storingData)
                {
                    Thread.Sleep(1);
                }

                try
                {
                    //Scan the Stream Store Queue's and if there are any shuffle them over to the bridge for synchronization:
                    DateTime? last = null;
                    for (var i = 0; i < Subscriptions.Count; i++)
                    {
                        BaseData data;
                        while (_streamStore[i].Queue.TryDequeue(out data))
                        {
                            last = data.Time;
                            Bridge[i].Enqueue(new List<BaseData> { data });
                            Log.Debug("LiveTradingDataFeed.Run(): Enqueuing Data... s:" + data.Symbol + " >> v:" + data.Value);
                        }
                    }

                    // if we dequeued someone, update frontier for live data sync on bridge
                    if (last.HasValue)
                    {
                        LoadedDataFrontier = last.Value;
                    }
                }
                catch (Exception err)
                {
                    Log.Error("LiveTradingDataFeed.Run(): " + err.Message);
                }

                //Prevent Thread Locking Up - Sleep 1ms (linux only, on windows will sleep 15ms).
                Thread.Sleep(1);

            } while (!_exitTriggered && !_endOfBridges);

            //Dispose of the realtime clock.
            realtime.Stop();

            //Stop thread
            _isActive = false;

            //Exiting RealTime Events:
            Log.Trace("LiveTradingDataFeed.Run(): Exiting Realtime Run Routine");
        }
Example #2
0
        /// <summary>
        /// Execute the primary thread for retrieving stock data.
        /// 1. Subscribe to the streams requested.
        /// 2. Build bars or tick data requested, primary loop increment smallest possible.
        /// </summary>
        public void Run()
        {
            // Symbols requested:
            _symbols = (from security in _algorithm.Securities.Values
                        where !security.IsDynamicallyLoadedData && (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
                        select security.Symbol).ToList<string>();

            //Initialize:
            _streamStore = new Dictionary<int, StreamStore>();
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                var config = _subscriptions[i];
                _streamStore.Add(i, new StreamStore(config, _algorithm.Securities[config.Symbol]));
            }
            Log.Trace(string.Format("LiveTradingDataFeed.Stream(): Initialized {0} stream stores.", _streamStore.Count));

            // Set up separate thread to handle stream and building packets:
            var streamThread = new Thread(StreamStoreConsumer);
            streamThread.Start();
            Thread.Sleep(5); // Wait a little for the other thread to init.

            var sourceDate = DateTime.Now.Date;
            var resumeRun = new ManualResetEvent(true);

            // This thread converts data into bars "on" the second - assuring the bars are close as
            // possible to a second unit tradebar (starting at 0 milliseconds).
            var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), triggerTime =>
            {
                //Pause bridge queing operations:
                resumeRun.Reset();

                // determine if we're on even time boundaries for data emit
                var onMinute = triggerTime.Second == 0;
                var onHour = onMinute && triggerTime.Minute == 0;
                var onDay = onHour && triggerTime.Hour == 0;

                if (triggerTime.Date != sourceDate)
                {
                    //Every day refresh the source file for the custom user data:
                    for (int i = 0; i < Subscriptions.Count; i++)
                    {
                        _subscriptionManagers[i].RefreshSource(triggerTime.Date);
                        sourceDate = triggerTime.Date;
                    }
                }

                // Determine if this subscription needs to be archived:
                for (var i = 0; i < Subscriptions.Count; i++)
                {

                    bool triggerArchive = false;
                    switch (_subscriptions[i].Resolution)
                    {
                        case Resolution.Second:
                            triggerArchive = true;
                            break;
                        case Resolution.Minute:
                            triggerArchive = onMinute;
                            break;
                        case Resolution.Hour:
                            triggerArchive = onHour;
                            break;
                        case Resolution.Daily:
                            triggerArchive = onDay;
                            break;
                    }

                    if (triggerArchive)
                    {
                        _streamStore[i].TriggerArchive(triggerTime, _subscriptions[i].FillDataForward);
                    }
                }
                //Resume bridge queing operations:
                resumeRun.Set();
            });

            //Start the realtime sampler above
            realtime.Start();

            while (!_exitTriggered && !_endOfBridges)
            {
                resumeRun.WaitOne();

                try
                {
                    //Scan the Stream Store Queue's and if there are any shuffle them over to the bridge for synchronization:
                    DateTime? last = null;
                    for (var i = 0; i < Subscriptions.Count; i++)
                    {
                        BaseData data;
                        while (_streamStore[i].Queue.TryDequeue(out data))
                        {
                            last = data.Time;
                            Bridge[i].Enqueue(new List<BaseData> { data });
                        }
                    }

                    // if we dequeued someone, update frontier for live data sync on bridge
                    if (last.HasValue)
                    {
                        LoadedDataFrontier = last.Value;
                    }
                }
                catch (Exception err)
                {
                    Log.Error("LiveTradingDataFeed.Run(): " + err.Message);
                }

                Thread.Sleep(1);
            }

            //Dispose of the realtime clock.
            realtime.Stop();

            //Stop thread
            _isActive = false;

            //Exit Live DataStream Feed:
            Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method");
        }
Example #3
0
        /// <summary>
        /// Execute the primary thread for retrieving stock data.
        /// 1. Subscribe to the streams requested.
        /// 2. Build bars or tick data requested, primary loop increment smallest possible.
        /// </summary>
        public void Run()
        {
            // Symbols requested:
            _symbols = (from security in _algorithm.Securities.Values
                        where !security.IsDynamicallyLoadedData && (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
                        select security.Symbol).ToList<string>();

            //Initialize:
            _streamStores = new Dictionary<int, StreamStore>();
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                var config = _subscriptions[i];
                if (config.Resolution != Resolution.Tick)
                {
                    _streamStores.Add(i, new StreamStore(config, _algorithm.Securities[config.Symbol]));
                }
            }
            Log.Trace(string.Format("LiveTradingDataFeed.Stream(): Initialized {0} stream stores.", _streamStores.Count));

            // Set up separate thread to handle stream and building packets:
            var streamThread = new Thread(StreamStoreConsumer);
            streamThread.Start();
            Thread.Sleep(5); // Wait a little for the other thread to init.

            // This thread converts data into bars "on" the second - assuring the bars are close as
            // possible to a second unit tradebar (starting at 0 milliseconds).
            var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime =>
            {
                // determine if we're on even time boundaries for data emit
                var onMinute = utcTriggerTime.Second == 0;
                var onHour = onMinute && utcTriggerTime.Minute == 0;
                var onDay = onHour && utcTriggerTime.Hour == 0;

                // Determine if this subscription needs to be archived:
                var items = new Dictionary<int, List<BaseData>>();
                for (var i = 0; i < Subscriptions.Count; i++)
                {
                    // stream stores are only created for non-tick data and this timer thread is used
                    // soley for dequeuing from the stream stores, this index, i, would be null
                    if (Subscriptions[i].Resolution == Resolution.Tick) continue;

                    bool triggerArchive = false;
                    switch (_subscriptions[i].Resolution)
                    {
                        case Resolution.Second:
                            triggerArchive = true;
                            break;
                        case Resolution.Minute:
                            triggerArchive = onMinute;
                            break;
                        case Resolution.Hour:
                            triggerArchive = onHour;
                            break;
                        case Resolution.Daily:
                            triggerArchive = onDay;
                            break;
                    }

                    if (triggerArchive)
                    {
                        _streamStores[i].TriggerArchive(utcTriggerTime, _subscriptions[i].FillDataForward);

                        BaseData data;
                        var dataPoints = new List<BaseData>();
                        while (_streamStores[i].Queue.TryDequeue(out data))
                        {
                            dataPoints.Add(data);
                        }
                        items[i] = dataPoints;
                    }
                }

                // don't try to add if we're already cancelling
                if (_cancellationTokenSource.IsCancellationRequested) return;
                Bridge.Add(new TimeSlice(utcTriggerTime, items));
            });

            //Start the realtime sampler above
            realtime.Start();

            while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges)
            {
                // main work of this class is done in the realtime and stream store consumer threads
                Thread.Sleep(1000);
            }

            //Dispose of the realtime clock.
            realtime.Stop();

            //Stop thread
            _isActive = false;

            //Exit Live DataStream Feed:
            Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method");
        }
Example #4
0
        /// <summary>
        /// Execute the primary thread for retrieving stock data.
        /// 1. Subscribe to the streams requested.
        /// 2. Build bars or tick data requested, primary loop increment smallest possible.
        /// </summary>
        public void Run()
        {
            //Initialize:

            // Set up separate thread to handle stream and building packets:
            var streamThread = new Thread(StreamStoreConsumer);
            streamThread.Start();
            Thread.Sleep(5); // Wait a little for the other thread to init.

            // This thread converts data into bars "on" the second - assuring the bars are close as
            // possible to a second unit tradebar (starting at 0 milliseconds).
            var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime =>
            {
                // determine if we're on even time boundaries for data emit
                var onMinute = utcTriggerTime.Second == 0;
                var onHour = onMinute && utcTriggerTime.Minute == 0;

                // Determine if this subscription needs to be archived:
                var items = new List<KeyValuePair<Security, List<BaseData>>>();

                var changes = SecurityChanges.None;

                var performedUniverseSelection = new HashSet<string>();
                foreach (var kvp in _subscriptions)
                {
                    var subscription = kvp.Value;

                    if (subscription.Configuration.Resolution == Resolution.Tick) continue;

                    var localTime = new DateTime(utcTriggerTime.Ticks - subscription.OffsetProvider.GetOffsetTicks(utcTriggerTime));
                    var onDay = onHour && localTime.Hour == 0;

                    // perform universe selection if requested on day changes (don't perform multiple times per market)
                    if (onDay && _algorithm.Universe != null && !performedUniverseSelection.Contains(subscription.Configuration.Symbol))
                    {
                        performedUniverseSelection.Add(subscription.Configuration.Symbol);
                        var coarse = UniverseSelection.GetCoarseFundamentals(subscription.Configuration.Market, subscription.TimeZone, localTime.Date, true);
                        changes = _universeSelection.ApplyUniverseSelection(localTime.Date, coarse);
                    }

                    var triggerArchive = false;
                    switch (subscription.Configuration.Resolution)
                    {
                        case Resolution.Second:
                            triggerArchive = true;
                            break;
                        case Resolution.Minute:
                            triggerArchive = onMinute;
                            break;
                        case Resolution.Hour:
                            triggerArchive = onHour;
                            break;
                        case Resolution.Daily:
                            triggerArchive = onDay;
                            break;
                    }

                    if (triggerArchive)
                    {
                        subscription.StreamStore.TriggerArchive(utcTriggerTime, subscription.Configuration.FillDataForward);

                        BaseData data;
                        var dataPoints = new List<BaseData>();
                        while (subscription.StreamStore.Queue.TryDequeue(out data))
                        {
                            dataPoints.Add(data);
                        }
                        items.Add(new KeyValuePair<Security, List<BaseData>>(subscription.Security, dataPoints));
                    }
                }

                // don't try to add if we're already cancelling
                if (_cancellationTokenSource.IsCancellationRequested) return;
                Bridge.Add(TimeSlice.Create(_algorithm, utcTriggerTime, items, changes));
            });

            //Start the realtime sampler above
            realtime.Start();

            while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges)
            {
                // main work of this class is done in the realtime and stream store consumer threads
                Thread.Sleep(1000);
            }

            //Dispose of the realtime clock.
            realtime.Stop();

            //Stop thread
            _isActive = false;

            //Exit Live DataStream Feed:
            Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method");
        }
Example #5
0
        /// <summary>
        /// Execute the primary thread for retrieving stock data.
        /// 1. Subscribe to the streams requested.
        /// 2. Build bars or tick data requested, primary loop increment smallest possible.
        /// </summary>
        public void Run()
        {
            //Initialize:

            // Set up separate thread to handle stream and building packets:
            var streamThread = new Thread(StreamStoreConsumer);
            streamThread.Start();
            Thread.Sleep(5); // Wait a little for the other thread to init.

            // This thread converts data into bars "on" the second - assuring the bars are close as
            // possible to a second unit tradebar (starting at 0 milliseconds).
            var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime =>
            {
                // determine if we're on even time boundaries for data emit
                var onMinute = utcTriggerTime.Second == 0;
                var onHour = onMinute && utcTriggerTime.Minute == 0;

                // Determine if this subscription needs to be archived:
                var items = new List<KeyValuePair<Security, List<BaseData>>>();

                var changes = SecurityChanges.None;

                foreach (var kvp in _subscriptions)
                {
                    var subscription = kvp.Value;

                    if (subscription.Configuration.Resolution == Resolution.Tick) continue;

                    var localTime = new DateTime(utcTriggerTime.Ticks - subscription.OffsetProvider.GetOffsetTicks(utcTriggerTime));
                    var onDay = onHour && localTime.Hour == 0;

                    var triggerArchive = false;
                    switch (subscription.Configuration.Resolution)
                    {
                        case Resolution.Second:
                            triggerArchive = true;
                            break;
                        case Resolution.Minute:
                            triggerArchive = onMinute;
                            break;
                        case Resolution.Hour:
                            triggerArchive = onHour;
                            break;
                        case Resolution.Daily:
                            triggerArchive = onDay;
                            break;
                    }

                    if (triggerArchive)
                    {
                        var data = subscription.StreamStore.TriggerArchive(utcTriggerTime);
                        if (data != null)
                        {
                            items.Add(new KeyValuePair<Security, List<BaseData>>(subscription.Security, new List<BaseData> {data}));
                        }
                    }
                }

                // don't try to add if we're already cancelling
                if (_cancellationTokenSource.IsCancellationRequested) return;
                Bridge.Add(TimeSlice.Create(utcTriggerTime, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, items, changes));
            });

            //Start the realtime sampler above
            realtime.Start();

            while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges)
            {
                // main work of this class is done in the realtime and stream store consumer threads
                Thread.Sleep(1000);
            }

            //Dispose of the realtime clock.
            realtime.Stop();

            //Stop thread
            _isActive = false;

            //Exit Live DataStream Feed:
            Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method");
        }