Example #1
0
        /********************************************************
        * CLASS CONSTRUCTOR
        *********************************************************/
        /// <summary>
        /// Create a new backtesting data feed.
        /// </summary>
        /// <param name="algorithm">Instance of the algorithm</param>
        /// <param name="job">Algorithm work task</param>
        public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
        {
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            DataFeed = DataFeedEndpoint.FileSystem;
            IsActive = true;
            Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
            EndOfBridge = new bool[_subscriptions];
            SubscriptionReaders = new SubscriptionDataReader[_subscriptions];
            FillForwardFrontiers = new DateTime[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);

            //Class Privates:
            _job = job;
            _algorithm = algorithm;
            _endOfStreams = false;
            _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:

            for (var i = 0; i < _subscriptions; i++)
            {
                //Create a new instance in the dictionary:
                Bridge[i] = new ConcurrentQueue<List<BaseData>>();
                EndOfBridge[i] = false;
                SubscriptionReaders[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish);
                FillForwardFrontiers[i] = new DateTime();
            }
        }
Example #2
0
        /********************************************************
        * CLASS CONSTRUCTOR
        *********************************************************/
        /// <summary>
        /// Create an instance of the base datafeed.
        /// </summary>
        public BaseDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
        {
            //Save the data subscriptions
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            DataFeed = DataFeedEndpoint.FileSystem;
            IsActive = true;
            Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
            EndOfBridge = new bool[_subscriptions];
            SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);
            _frontierTime = new DateTime[_subscriptions];

            //Class Privates:
            _job = job;
            _algorithm = algorithm;
            _endOfStreams = false;
            _bridgeMax = _bridgeMax / _subscriptions;

            //Initialize arrays:
            for (var i = 0; i < _subscriptions; i++)
            {
                _frontierTime[i] = job.PeriodStart;
                EndOfBridge[i] = false;
                Bridge[i] = new ConcurrentQueue<List<BaseData>>();
                SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, job.PeriodStart, job.PeriodFinish);
            }
        }
Example #3
0
        private static LiveSubscription CreateSubscription(IAlgorithm algorithm,
            IResultHandler resultHandler,
            Security security,
            DateTime periodStart,
            DateTime periodEnd)
        {
            IEnumerator<BaseData> enumerator = null;
            if (security.IsDynamicallyLoadedData)
            {
                //Subscription managers for downloading user data:
                // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
                var subscriptionDataReader = new SubscriptionDataReader(
                    security.SubscriptionDataConfig,
                    security,
                    periodStart, Time.EndOfTime,
                    resultHandler,
                    Time.EachTradeableDay(algorithm.Securities.Values, periodStart, periodEnd),
                    true,
                    DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).Date
                    );

                // wrap the subscription data reader with a filter enumerator
                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, periodEnd);
            }
            return new LiveSubscription(security, enumerator, periodStart, periodEnd, true, false);
        }
Example #4
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config);

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator<BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                    .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                var cf = new CoarseFundamental();

                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // load coarse data day by day
                var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
                             let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
                             let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
                             let source = cf.GetSource(config, dateInDataTimeZone, false)
                             let coarseFundamentalForDate = factory.Read(source)
                             select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);

                
                ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);

                enumerator = enqueueable;
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
Example #5
0
        private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
        {
            var config = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return null;
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;
            if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);

            // ReSharper disable once PossibleMultipleEnumeration
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
                    security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
            return subscription;
        }
Example #6
0
        /// <summary>
        /// Prepare and create the new MySQL Database connection datafeed.
        /// </summary>
        /// <param name="algorithm"></param>
        /// <param name="job"></param>
        /// <param name="resultHandler"></param>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
        {
            //Save the data subscriptions
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            IsActive = true;
            DataFeed = DataFeedEndpoint.FileSystem;
            Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
            EndOfBridge = new bool[_subscriptions];
            SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);
            _mySQLBridgeTime = new DateTime[_subscriptions];

            //Class Privates:
            _endOfStreams = false;
            _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
            _endTime = algorithm.EndDate;

            //Initialize arrays:
            for (var i = 0; i < _subscriptions; i++)
            {
                _mySQLBridgeTime[i] = algorithm.StartDate;
                EndOfBridge[i] = false;
                Bridge[i] = new ConcurrentQueue<List<BaseData>>();
                SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, algorithm.StartDate, algorithm.EndDate, resultHandler);
            }
        }
Example #7
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
        {
            Subscription subscription = null;
            try
            {
                var config = security.SubscriptionDataConfig;
                var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
                var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);

                IEnumerator<BaseData> enumerator;
                if (config.IsCustomData)
                {
                    // custom data uses backtest readers
                    var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
                    var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);

                    // apply fast forwarding, this is especially important for RemoteFile types that
                    // can send in large chunks of old, irrelevant data
                    var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);

                    // apply rate limits (1x per increment, max 30 minutes between calls)
                    // TODO : Pull limits from config file?
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));

                    // add the enumerator to the exchange
                    _customExchange.AddEnumerator(rateLimit);

                    // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
                    var enqueable = new EnqueableEnumerator<BaseData>();
                    _customExchange.SetHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueableEnumerator<BaseData>();
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);

                // finally, make our subscriptions aware of the frontier of the data feed, this will help
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }
Example #8
0
 /********************************************************
 * CLASS METHODS
 *********************************************************/
 /// <summary>
 /// Initialize activators to invoke types in the algorithm
 /// </summary>
 private void ResetActivators()
 {
     for (var i = 0; i < _subscriptions; i++)
     {
         //Create a new instance in the dictionary:
         Bridge[i] = new ConcurrentQueue<List<BaseData>>();
         EndOfBridge[i] = false;
         SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish);
         FillForwardFrontiers[i] = new DateTime();
     }
 }
Example #9
0
        /********************************************************
        * CLASS CONSTRUCTOR
        *********************************************************/
        /// <summary>
        /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
        /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
        /// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
        /// </summary>
        /// <param name="algorithm">Algorithm requesting data</param>
        protected LiveTradingDataFeed(IAlgorithm algorithm)
        {
            //Subscription Count:
            _subscriptions = algorithm.SubscriptionManager.Subscriptions;

            //Set Properties:
            _dataFeed = DataFeedEndpoint.LiveTrading;
            _isActive = true;
            _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
            _endOfBridge = new bool[Subscriptions.Count];
            _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
            _realtimePrices = new List<decimal>();

            //Class Privates:
            _algorithm = algorithm;

            //Setup the arrays:
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                _endOfBridge[i] = false;
                _bridge[i] = new ConcurrentQueue<List<BaseData>>();

                //This is quantconnect data source, store here for speed/ease of access
                _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);

                //Subscription managers for downloading user data:
                _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);

                //Set up the source file for today:
                _subscriptionManagers[i].RefreshSource(DateTime.Now.Date);

                _realtimePrices.Add(0);
            }
        }
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator<BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
                    .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
                    .Select(dt => new Tick {Time = dt}).GetEnumerator();
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false);
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);

            // only message the user if it's one of their universe types
            var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
            PrimeSubscriptionPump(subscription, messageUser);
            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
Example #11
0
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
        {
            _cancellationTokenSource = new CancellationTokenSource();
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            IsActive = true;
            _endOfBridge = new bool[_subscriptions];
            SubscriptionReaders = new IEnumerator<BaseData>[_subscriptions];
            FillForwardFrontiers = new DateTime[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);

            //Class Privates:
            _algorithm = algorithm;

            // find the minimum resolution, ignoring ticks
            var fillForwardResolution = Subscriptions
                .Where(x => x.Resolution != Resolution.Tick)
                .Select(x => x.Resolution.ToTimeSpan())
                .DefaultIfEmpty(TimeSpan.FromSeconds(1))
                .Min();

            // figure out how many subscriptions are at the minimum resolution
            var subscriptionsAtMinimumResolution =
                (from sub in Subscriptions
                 where sub.Resolution == Subscriptions.Min(x => x.Resolution)
                 select sub).Count();

            Bridge = new BlockingCollection<TimeSlice>(Math.Min(1000, 50000/subscriptionsAtMinimumResolution));

            for (var i = 0; i < _subscriptions; i++)
            {
                _endOfBridge[i] = false;

                var config = Subscriptions[i];
                var start = algorithm.StartDate;
                var end = algorithm.EndDate;
                var security = _algorithm.Securities[Subscriptions[i].Symbol];

                var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);

                IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security,
                    start, end,
                    resultHandler,
                    tradeableDates,
                    false,
                    // we want a backtest to use the same symbols regardless of start/end dates, so we set the resolution
                    // to null which means give me the current/most recent symbol mapping for the requested symbol
                    symbolResolutionDate: null
                    );

                // optionally apply fill forward logic, but never for tick data
                if (config.FillDataForward && config.Resolution != Resolution.Tick)
                {
                    enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
                }

                // finally apply exchange/user filters
                SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
                FillForwardFrontiers[i] = new DateTime();

                // prime the pump for iteration in Run
                _endOfBridge[i] = !SubscriptionReaders[i].MoveNext();

                if (_endOfBridge[i])
                {
                    Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol);
                }
            }
        }
Example #12
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(
            IUniverse universe,
            DateTime startTimeUtc,
            DateTime endTimeUtc
            )
        {
            // grab the relevant exchange hours
            SubscriptionDataConfig config = universe.Configuration;

            var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
                .GetExchangeHours(config.Market, null, config.SecurityType);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);

            // define our data enumerator
            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
            var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false);

            // create the subscription
            var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc);

            // only message the user if it's one of their universe types
            _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
        }
Example #13
0
        private static Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
        {
            var config = security.SubscriptionDataConfig;
            var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
            var symbolResolutionDate = userDefined ? (DateTime?)null : start;
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
                    security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
            var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
            return subscription;
        }
Example #14
0
        private void ProcessMissingDayFillForward(SubscriptionDataReader manager, int i, TimeSpan increment, DateTime dateToFill)
        {
            // we'll copy the current into the next day
            var subscription = Subscriptions[i];
            if (!subscription.FillDataForward || manager.Current == null) return;

            var start = dateToFill.Date + manager.Exchange.MarketOpen;
            if (subscription.ExtendedMarketHours)
            {
                // we need to loop to find the extended market open
                start = dateToFill.Date;
                while (!manager.Exchange.DateTimeIsExtendedOpen(start))
                {
                    start += increment;
                }
            }

            var current = manager.Current;
            for (var date = start; date.Date == dateToFill.Date; date = date + increment)
            {
                if (manager.IsMarketOpen(date) || (subscription.ExtendedMarketHours && manager.IsExtendedMarketOpen(date)))
                {
                    var cache = new List<BaseData>(1);
                    var fillforward = current.Clone(true);
                    fillforward.Time = date;
                    FillForwardFrontiers[i] = date;
                    cache.Add(fillforward);
                    Bridge[i].Enqueue(cache);
                }
                else
                {
                    // stop fill forwarding when we're no longer open
                    break;
                }
            }
        }
Example #15
0
        private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
        {
            var config = security.SubscriptionDataConfig;
            var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                if (userDefined)
                {
                    _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                }
                return null;
            }

            // ReSharper disable once PossibleMultipleEnumeration
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
                    security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
            var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
            return subscription;
        }
Example #16
0
        /// <summary>
        /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
        /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
        /// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
        {
            //Subscription Count:
            _subscriptions = algorithm.SubscriptionManager.Subscriptions;

            //Set Properties:
            _isActive = true;
            _dataFeed = DataFeedEndpoint.LiveTrading;
            _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
            _endOfBridge = new bool[Subscriptions.Count];
            _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
            _realtimePrices = new List<decimal>();

            //Set the source of the live data:
            _dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));

            //Class Privates:
            _algorithm = algorithm;
            if (!(job is LiveNodePacket))
            {
                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
            }

            _job = (LiveNodePacket) job;

            //Setup the arrays:
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                _endOfBridge[i] = false;
                _bridge[i] = new ConcurrentQueue<List<BaseData>>();

                //This is quantconnect data source, store here for speed/ease of access
                _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);

                //Subscription managers for downloading user data:
                _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue, resultHandler);

                //Set up the source file for today:
                _subscriptionManagers[i].RefreshSource(DateTime.Now.Date);

                _realtimePrices.Add(0);
            }

            // request for data from these symbols
            var symbols = BuildTypeSymbolList(algorithm);
            if (symbols.Any())
            {
                // don't subscribe if there's nothing there, this allows custom data to
                // work without an IDataQueueHandler implementation by specifying LiveDataQueue
                // in the configuration, that implementation throws on every method, but we actually
                // don't need it if we're only doing custom data
                _dataQueue.Subscribe(_job, symbols);
            }
        }
Example #17
0
        /// <summary>
        /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
        /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
        /// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
        {
            _cancellationTokenSource = new CancellationTokenSource();

            //Subscription Count:
            _subscriptions = algorithm.SubscriptionManager.Subscriptions;

            Bridge = new BlockingCollection<TimeSlice>();

            //Set Properties:
            _isActive = true;
            _endOfBridge = new bool[Subscriptions.Count];
            _subscriptionManagers = new IEnumerator<BaseData>[Subscriptions.Count];
            _realtimePrices = new List<decimal>();

            //Set the source of the live data:
            _dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));

            //Class Privates:
            _algorithm = algorithm;
            if (!(job is LiveNodePacket))
            {
                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
            }

            _job = (LiveNodePacket) job;

            //Setup the arrays:
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                _endOfBridge[i] = false;

                //This is quantconnect data source, store here for speed/ease of access
                var security = algorithm.Securities[_subscriptions[i].Symbol];
                _isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData);

                // only make readers for custom data, live data will come through data queue handler
                if (_isDynamicallyLoadedData[i])
                {
                    //Subscription managers for downloading user data:
                    // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
                    var periodStart = DateTime.Today.AddDays(-7);
                    var subscriptionDataReader = new SubscriptionDataReader(
                        _subscriptions[i],
                        security,
                        periodStart,
                        DateTime.MaxValue,
                        resultHandler,
                        Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue),
                        true,
                        DateTime.Today
                        );

                    // wrap the subscription data reader with a filter enumerator
                    _subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue);
                }

                _realtimePrices.Add(0);
            }

            // request for data from these symbols
            var symbols = BuildTypeSymbolList(algorithm);
            if (symbols.Any())
            {
                // don't subscribe if there's nothing there, this allows custom data to
                // work without an IDataQueueHandler implementation by specifying LiveDataQueue
                // in the configuration, that implementation throws on every method, but we actually
                // don't need it if we're only doing custom data
                _dataQueue.Subscribe(_job, symbols);
            }
        }
Example #18
0
        /// <summary>
        /// If this is a fillforward subscription, look at the previous time, and current time, and add new 
        /// objects to queue until current time to fill up the gaps.
        /// </summary>
        /// <param name="manager">Subscription to process</param>
        /// <param name="i">Subscription position in the bridge ( which queue are we pushing data to )</param>
        /// <param name="increment">Timespan increment to jump the fillforward results</param>
        void ProcessFillForward(SubscriptionDataReader manager, int i, TimeSpan increment)
        {
            // If previous == null cannot fill forward nothing there to move forward (e.g. cases where file not found on first file).
            if (!Subscriptions[i].FillDataForward || manager.Previous == null) return;

            //Last tradebar and the current one we're about to add to queue:
            var previous = manager.Previous;
            var current = manager.Current;

            //Initialize the frontier:
            if (FillForwardFrontiers[i].Ticks == 0) FillForwardFrontiers[i] = previous.Time;

            //Data ended before the market closed: premature ending flag - continue filling forward until market close.
            if (manager.EndOfStream && manager.MarketOpen(current.Time))
            {
                //Premature end of stream: fill manually until market closed.
                for (var date = FillForwardFrontiers[i] + increment; manager.MarketOpen(date); date = date + increment)
                {
                    var cache = new List<BaseData>(1);
                    var fillforward = current.Clone(true);
                    fillforward.Time = date;
                    FillForwardFrontiers[i] = date;
                    cache.Add(fillforward);
                    Bridge[i].Enqueue(cache);
                }
                return;
            }

            //Once per increment, add a new cache to the Bridge:
            //If the current.Time is before market close, (e.g. suspended trading at 2pm) the date is always greater than currentTime and fillforward never runs.
            //In this circumstance we need to keep looping till market/extended hours close even if no data.
            for (var date = FillForwardFrontiers[i] + increment; (date < current.Time); date = date + increment)
            {
                //If we don't want aftermarket data, rewind it backwards until the market closes.
                if (!Subscriptions[i].ExtendedMarketHours)
                {
                    if (!manager.MarketOpen(date))
                    {
                        // Move fill forward so we don't waste time in this tight loop.
                        //Question is where to shuffle the date?
                        // --> If BEFORE market open, shuffle forward.
                        // --> If AFTER market close, and current.Time after market close, quit loop.
                        date = current.Time;
                        do
                        {
                            date = date - increment;
                        } while (manager.MarketOpen(date));
                        continue;
                    }
                }
                else
                {
                    //If we've asked for extended hours, and the security is no longer inside extended market hours, skip:
                    if (!manager.ExtendedMarketOpen(date))
                    {
                        continue;
                    }
                }

                var cache = new List<BaseData>(1);
                var fillforward = previous.Clone(true);
                fillforward.Time = date;
                FillForwardFrontiers[i] = date;
                cache.Add(fillforward);
                Bridge[i].Enqueue(cache);
            }
        }
Example #19
0
        /// <summary>
        /// Prepare and create the new MySQL Database connection datafeed.
        /// </summary>
        /// <param name="algorithm"></param>
        /// <param name="job"></param>
        /// <param name="resultHandler"></param>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
        {
            Bridge = new BlockingCollection<TimeSlice>();

            //Save the data subscriptions
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            IsActive = true;
            EndOfBridge = new bool[_subscriptions];
            _subscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);
            _mySQLBridgeTime = new DateTime[_subscriptions];

            //Class Privates:
            _endTime = algorithm.EndDate;

            //Initialize arrays:
            for (var i = 0; i < _subscriptions; i++)
            {
                _mySQLBridgeTime[i] = algorithm.StartDate;
                EndOfBridge[i] = false;
                _subscriptionReaderManagers[i] = new SubscriptionDataReader(
                    Subscriptions[i],
                    algorithm.Securities[Subscriptions[i].Symbol],
                    algorithm.StartDate,
                    algorithm.EndDate,
                    resultHandler,
                    Time.EachTradeableDay(algorithm.Securities, algorithm.StartDate, algorithm.EndDate),
                    job is LiveNodePacket
                    );
            }
        }
Example #20
0
        /********************************************************
        * CLASS CONSTRUCTOR
        *********************************************************/
        /// <summary>
        /// Create a new backtesting data feed.
        /// </summary>
        /// <param name="algorithm">Instance of the algorithm</param>
        /// <param name="job">Algorithm work task</param>
        public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
        {
            Subscriptions = algorithm.SubscriptionManager.Subscriptions;
            _subscriptions = Subscriptions.Count;

            //Public Properties:
            DataFeed = DataFeedEndpoint.FileSystem;
            IsActive = true;
            Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
            EndOfBridge = new bool[_subscriptions];
            SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
            FillForwardFrontiers = new DateTime[_subscriptions];
            RealtimePrices = new List<decimal>(_subscriptions);

            //Class Privates:
            _job = job;
            _algorithm = algorithm;
            _endOfStreams = false;
            _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
        }
        /// <summary>
        /// Creates an enumerator for the specified security/configuration
        /// </summary>
        private IEnumerator<BaseData> CreateSubscriptionEnumerator(Security security,
            SubscriptionDataConfig config,
            DateTime localStartTime,
            DateTime localEndTime,
            MapFileResolver mapFileResolver,
            IEnumerable<DateTime> tradeableDates,
            bool applySubscripterFilterEnumerator = true)
        {
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
                _factorFileProvider, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
                    security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // optionally apply exchange/user filters
            if (applySubscripterFilterEnumerator)
            {
                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
            }
            return enumerator;
        }
Example #22
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(
            IUniverse universe,
            DateTime startTimeUtc,
            DateTime endTimeUtc
            )
        {
            // grab the relevant exchange hours
            var config = universe.Configuration;

            var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
                .GetExchangeHours(config.Market, null, config.SecurityType);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            IEnumerator<BaseData> enumerator;
            if (config.Type == typeof (CoarseFundamental))
            {
                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>>
                {
                    {config.SecurityType, new List<string>{config.Symbol}}
                });

                var enqueable = new EnqueableEnumerator<BaseData>();
                _exchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                });
                enumerator = enqueable;
            }
            else
            {
                var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
                var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);

                // define our data enumerator
                var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
                var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true);
                _customExchange.AddEnumerator(reader);

                var enqueable = new EnqueableEnumerator<BaseData>();
                _customExchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                    else
                    {
                        enqueable.Enqueue(data);
                    }
                });
                enumerator = enqueable;
            }

            // create the subscription
            var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc);

            return subscription;
        }
Example #23
0
        /// <summary>
        /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
        /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
        /// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
        /// </summary>
        public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
        {
            //Subscription Count:
            _subscriptions = algorithm.SubscriptionManager.Subscriptions;

            //Set Properties:
            _isActive = true;
            _dataFeed = DataFeedEndpoint.LiveTrading;
            _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
            _endOfBridge = new bool[Subscriptions.Count];
            _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
            _realtimePrices = new List<decimal>();

            //Set the source of the live data:
            _dataQueue = dataSource;

            //Class Privates:
            _algorithm = algorithm;
            _job = job;

            //Setup the arrays:
            for (var i = 0; i < Subscriptions.Count; i++)
            {
                _endOfBridge[i] = false;
                _bridge[i] = new ConcurrentQueue<List<BaseData>>();

                //This is quantconnect data source, store here for speed/ease of access
                _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);

                //Subscription managers for downloading user data:
                _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);

                //Set up the source file for today:
                _subscriptionManagers[i].RefreshSource(DateTime.Now.Date);

                _realtimePrices.Add(0);
            }

            // request for data from these symbols
            _dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
        }