/// <summary> /// Get the last known price using the history provider. /// Useful for seeding securities with the correct price /// </summary> /// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param> /// <returns>A single <see cref="BaseData"/> object with the last known price</returns> public BaseData GetLastKnownPrice(Security security) { if (security.Symbol.IsCanonical() || HistoryProvider == null) { return(null); } var configs = SubscriptionManager.SubscriptionDataConfigService .GetSubscriptionDataConfigs(security.Symbol); // For speed and memory usage, use Resolution.Minute as the minimum resolution var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)configs.GetHighestResolution()); var isExtendedMarketHours = configs.IsExtendedMarketHours(); var startTime = _historyRequestFactory.GetStartTimeAlgoTz(security.Symbol, 1, resolution, security.Exchange.Hours); var endTime = Time.RoundDown(resolution.ToTimeSpan()); // request QuoteBar for Options and Futures var dataType = typeof(BaseData); if (security.Type == SecurityType.Option || security.Type == SecurityType.Future) { dataType = LeanData.GetDataType(resolution, TickType.Quote); } // Get the config with the largest resolution var subscriptionDataConfig = GetMatchingSubscription(security.Symbol, dataType); // if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar if (subscriptionDataConfig != null && subscriptionDataConfig.Resolution == Resolution.Tick) { dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType); subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution); } var request = new HistoryRequest( startTime.ConvertToUtc(_localTimeKeeper.TimeZone), endTime.ConvertToUtc(_localTimeKeeper.TimeZone), subscriptionDataConfig == null ? typeof(TradeBar) : subscriptionDataConfig.Type, security.Symbol, resolution, security.Exchange.Hours, MarketHoursDatabase.FromDataFolder().GetDataTimeZone(security.Symbol.ID.Market, security.Symbol, security.Symbol.SecurityType), resolution, isExtendedMarketHours, configs.IsCustomData(), configs.DataNormalizationMode(), subscriptionDataConfig == null ? LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), security.Type) : subscriptionDataConfig.TickType ); var history = History(new List <HistoryRequest> { request }).ToList(); if (history.Any() && history.First().Values.Any()) { return(history.First().Values.First()); } return(null); }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is ITimeTriggeredUniverse) { var universe = request.Universe as UserDefinedUniverse; if (universe != null) { // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_algorithm.UtcTime, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection); _algorithm.OnSecuritiesChanged(changes); }; } return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider)); } if (request.Universe is FuturesChainUniverse) { return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } return(_subscriptionFactory); }
public void InitializesFromDataFolder() { var provider = MarketHoursDatabase.FromDataFolder(); Assert.AreNotEqual(0, provider.ExchangeHoursListing.Count); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _dataProvider = dataProvider; _dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new SubscriptionCollection(); _bridge = new BusyBlockingCollection <TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); // Not sure if this is needed but left here because of this: // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678 UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm) { var feed = new MockDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService( algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)); algorithm.Securities.SetSecurityService(securityService); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(feed, new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()), algorithm, algorithm.TimeKeeper, marketHoursDatabase, false, RegisteredSecurityDataTypesProvider.Null, dataPermissionManager); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.Initialize(); algorithm.PostInitialize(); // set exchanges to be always open foreach (var kvp in algorithm.Securities) { var security = kvp.Value; security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone)); } var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork); var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork); var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager); var timeSliceFactory = new TimeSliceFactory(algorithm.TimeZone); var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection); synchronizer.SetTimeProvider(subscriptionBasedTimeProvider); synchronizer.SetTimeSliceFactory(timeSliceFactory); var totalDataPoints = 0; var subscriptions = dataManager.DataFeedSubscriptions; foreach (var kvp in algorithm.Securities) { int dataPointCount; subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount)); totalDataPoints += dataPointCount; } // log what we're doing Log.Trace($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}"); var count = 0; DateTime currentTime = DateTime.MaxValue; DateTime previousValue; var stopwatch = Stopwatch.StartNew(); var enumerator = synchronizer.Sync(subscriptions, CancellationToken.None).GetEnumerator(); do { previousValue = currentTime; enumerator.MoveNext(); var timeSlice = enumerator.Current; currentTime = timeSlice.Time; count += timeSlice.DataPointCount; }while (currentTime != previousValue); stopwatch.Stop(); enumerator.DisposeSafely(); var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds; Log.Trace($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}"); Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose dataManager.RemoveAllSubscriptions(); }
public void TestBasicFeaturesWithOptionsFutures() { var securities = new SecurityManager(TimeKeeper); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); securities.Add( Symbols.SPY, new Security( SecurityExchangeHours, CreateTradeBarDataConfig(SecurityType.Equity, Symbols.SPY), new Cash(Currencies.USD, 0, 1m), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ) ); securities[Symbols.SPY].SetMarketPrice(new TradeBar { Time = securities.UtcTime, Symbol = Symbols.SPY, Close = 195 }); var option1 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 192m, new DateTime(2016, 02, 16)); securities.Add( option1, new Option( SecurityExchangeHours, CreateTradeBarDataConfig(SecurityType.Option, option1), new Cash(Currencies.USD, 0, 1m), new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ) ); var option2 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 193m, new DateTime(2016, 03, 19)); securities.Add( option2, new Option( SecurityExchangeHours, CreateTradeBarDataConfig(SecurityType.Option, option2), new Cash(Currencies.USD, 0, 1m), new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ) ); var future1 = Symbol.CreateFuture("ES", Market.USA, new DateTime(2016, 02, 16)); securities.Add( future1, new Future( SecurityExchangeHours, CreateTradeBarDataConfig(SecurityType.Future, future1), new Cash(Currencies.USD, 0, 1m), new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ) ); var future2 = Symbol.CreateFuture("ES", Market.USA, new DateTime(2016, 02, 19)); securities.Add( future2, new Future( SecurityExchangeHours, CreateTradeBarDataConfig(SecurityType.Future, future2), new Cash(Currencies.USD, 0, 1m), new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ) ); var cal = new TradingCalendar(securities, marketHoursDatabase); var optionDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count(); Assert.AreEqual(2, optionDays); var futureDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count(); Assert.AreEqual(2, futureDays); var days = cal.GetTradingDays(new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)); var optionAndfutureDays = days.Where(x => x.FutureExpirations.Any() || x.OptionExpirations.Any()).Count(); Assert.AreEqual(3, optionAndfutureDays); // why? because option1 and future1 expire in one day 2016-02-16. Lets have a look. var day = cal.GetTradingDay(new DateTime(2016, 02, 16)); Assert.AreEqual(1, day.OptionExpirations.Count()); Assert.AreEqual(1, day.FutureExpirations.Count()); var businessDays = days.Where(x => x.BusinessDay).Count(); Assert.AreEqual(24, businessDays); var weekends = days.Where(x => x.Weekend).Count(); Assert.AreEqual(9, weekends); Assert.AreEqual(24 + 9, (new DateTime(2016, 03, 19) - new DateTime(2016, 02, 16)).TotalDays + 1 /*inclusive*/); }
public void RefreshesOptionChainUniverseOnDateChange() { var startTime = new DateTime(2018, 10, 19, 10, 0, 0); var timeProvider = new ManualTimeProvider(startTime); var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.NewYork, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var option = new Option( canonicalSymbol, exchangeHours, quoteCurrency, new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache(), null ); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); EnqueueableEnumerator <BaseData> underlyingEnumerator = null; Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req) => { underlyingEnumerator = new EnqueueableEnumerator <BaseData>(); return(new LiveFillForwardEnumerator( timeProvider, underlyingEnumerator, option.Exchange, fillForwardResolution, false, Time.EndOfTime, Resolution.Minute.ToTimeSpan(), TimeZones.Utc)); }; var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider); var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero); var universe = new OptionChainUniverse(option, universeSettings, true); var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime); var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, TestGlobals.DataProvider); // 2018-10-19 10:00 AM UTC underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); // 2018-10-19 10:01 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); var data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-19 10:02 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-19 10:03 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-20 10:03 AM UTC timeProvider.Advance(Time.OneDay); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-20 10:04 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); Assert.IsNotNull(data.Underlying); enumerator.Dispose(); }
/// <summary> /// Creates an instance of the PortfolioLooper class /// </summary> /// <param name="startingCash">Equity curve</param> /// <param name="orders">Order events</param> /// <param name="resolution">Optional parameter to override default resolution (Hourly)</param> private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution) { // Initialize the providers that the HistoryProvider requires var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider"); var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider"); var dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false); var historyProvider = Composer.Instance.GetExportedValueByTypeName <IHistoryProvider>("SubscriptionDataReaderHistoryProvider"); historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, dataCacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false)); Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders); Algorithm.SetHistoryProvider(historyProvider); // Dummy LEAN datafeed classes and initializations that essentially do nothing var job = new BacktestNodePacket(1, 2, "3", null, 9m, $""); var feed = new MockDataFeed(); // Create MHDB and Symbol properties DB instances for the DataManager var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); _dataManager = new DataManager(feed, new UniverseSelection( Algorithm, new SecurityService(Algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(Algorithm.Portfolio))), Algorithm, Algorithm.TimeKeeper, marketHoursDatabase, false, RegisteredSecurityDataTypesProvider.Null); _securityService = new SecurityService(Algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(Algorithm.Portfolio)); var transactions = new BacktestingTransactionHandler(); var results = new BacktestingResultHandler(); // Initialize security services and other properties so that we // don't get null reference exceptions during our re-calculation Algorithm.Securities.SetSecurityService(_securityService); Algorithm.SubscriptionManager.SetDataManager(_dataManager); // Initializes all the proper Securities from the orders provided by the user Algorithm.FromOrders(orders); // Initialize the algorithm Algorithm.Initialize(); Algorithm.PostInitialize(); // More initialization, this time with Algorithm and other misc. classes results.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions); results.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue); transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), results); feed.Initialize(Algorithm, job, results, null, null, null, _dataManager, null); // Begin setting up the currency conversion feed if needed var coreSecurities = Algorithm.Securities.Values.ToList(); if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto)) { BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection); var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList(); // Skip the history request if we don't need to convert anything if (conversionSecurities.Any()) { // Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency _conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution); } } }
public void DoesNotEmitInvalidData() { var startTime = new DateTime(2014, 06, 06, 0, 0, 0); var endTime = new DateTime(2014, 06, 09, 20, 0, 0); var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.NewYork, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var option = new Option( canonicalSymbol, exchangeHours, quoteCurrency, new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req, input) => { input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol); return(new FillForwardEnumerator( input, option.Exchange, fillForwardResolution, false, endTime, Resolution.Minute.ToTimeSpan(), TimeZones.Utc, startTime)); }; var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider())); var request = new SubscriptionRequest(true, null, option, config, startTime, endTime); var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider()); var emittedCount = 0; foreach (var data in enumerator.AsEnumerable()) { emittedCount++; var optionData = data as OptionChainUniverseDataCollection; Assert.IsNotNull(optionData); Assert.IsNotNull(optionData.Underlying); Assert.AreNotEqual(0, optionData.Data.Count); } // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780 Assert.AreEqual(780, emittedCount); }
/// <summary> /// Initializes a new instance of the <see cref="CryptoExchange"/> class using market hours /// derived from the market-hours-database for the Crypto market /// </summary> public CryptoExchange(string market) : base(MarketHoursDatabase.FromDataFolder().GetExchangeHours(market, null, SecurityType.Crypto, TimeZones.Utc)) { }
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm) { var feed = new AlgorithmManagerTests.MockDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService( algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm); algorithm.Securities.SetSecurityService(securityService); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm, securityService), algorithm.Settings, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.Initialize(); algorithm.PostInitialize(); // set exchanges to be always open foreach (var kvp in algorithm.Securities) { var security = kvp.Value; security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone)); } var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork); var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork); var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager); var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection, algorithm.Portfolio.CashBook); synchronizer.SetTimeProvider(subscriptionBasedTimeProvider); synchronizer.SetSliceTimeZone(algorithm.TimeZone); var totalDataPoints = 0; var subscriptions = dataManager.DataFeedSubscriptions; foreach (var kvp in algorithm.Securities) { int dataPointCount; subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount)); totalDataPoints += dataPointCount; } // force JIT synchronizer.Sync(subscriptions); // log what we're doing Console.WriteLine($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}"); var count = 0; DateTime currentTime = DateTime.MaxValue; DateTime previousValue; var stopwatch = Stopwatch.StartNew(); do { previousValue = currentTime; var timeSlice = synchronizer.Sync(subscriptions); currentTime = timeSlice.Time; count += timeSlice.DataPointCount; }while (currentTime != previousValue); stopwatch.Stop(); var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds; Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}"); Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { Subscription subscription = null; // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol}"); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // we subscribe using a normalized symbol, without a random GUID, // since the ticker plant will send the coarse data using this symbol var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); // We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data _exchange.AddDataHandler(normalizedSymbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); }); enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider, // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider); return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
/// <summary> /// Downloads historical data from the brokerage and saves it in LEAN format. /// </summary> /// <param name="brokerage">The brokerage from where to fetch the data</param> /// <param name="symbols">The list of symbols</param> /// <param name="startTimeUtc">The starting date/time (UTC)</param> /// <param name="endTimeUtc">The ending date/time (UTC)</param> public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc) { if (symbols.Count == 0) { throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty."); } if (_tickType != TickType.Trade && _tickType != TickType.Quote) { throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote."); } if (symbols.Any(x => x.SecurityType != _securityType)) { throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type."); } if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1) { throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker."); } var dataType = LeanData.GetDataType(_resolution, _tickType); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var ticker = symbols.First().ID.Symbol; var market = symbols.First().ID.Market; var canonicalSymbol = Symbol.Create(ticker, _securityType, market); var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType); var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType); foreach (var symbol in symbols) { var historyRequest = new HistoryRequest( startTimeUtc, endTimeUtc, dataType, symbol, _resolution, exchangeHours, dataTimeZone, _resolution, true, false, DataNormalizationMode.Raw, _tickType ); var history = brokerage.GetHistory(historyRequest) .Select( x => { // Convert to date timezone before we write it x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone); return(x); }) .ToList(); // Generate a writer for this data and write it var writer = new LeanDataWriter(_resolution, symbol, _dataDirectory, _tickType); writer.Write(history); } }
public void FastExitsDoNotThrowUnhandledExceptions() { var algorithm = new AlgorithmStub(); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); var feed = new TestableLiveTradingDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService( algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm); algorithm.Securities.SetSecurityService(securityService); var dataManager = new DataManager(feed, new UniverseSelection(algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); var synchronizer = new TestableSynchronizer(_algorithm, dataManager, true); algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList()); var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick { Symbol = SymbolCache.GetSymbol(x.ToString()) }).ToList(); feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); feed.Initialize( algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager, synchronizer); var unhandledExceptionWasThrown = false; try { feed.Exit(); } catch (Exception ex) { QuantConnect.Logging.Log.Error(ex.ToString()); unhandledExceptionWasThrown = true; } Thread.Sleep(500); Assert.IsFalse(unhandledExceptionWasThrown); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return(subscription); }
/// <summary> /// Downloads historical data from the brokerage and saves it in LEAN format. /// </summary> /// <param name="brokerage">The brokerage from where to fetch the data</param> /// <param name="symbols">The list of symbols</param> /// <param name="startTimeUtc">The starting date/time (UTC)</param> /// <param name="endTimeUtc">The ending date/time (UTC)</param> public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc) { if (symbols.Count == 0) { throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty."); } if (_tickType != TickType.Trade && _tickType != TickType.Quote) { throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote."); } if (_securityType != SecurityType.Future && _securityType != SecurityType.Option && _securityType != SecurityType.FutureOption) { throw new ArgumentException($"DownloadAndSave(): The security type must be {SecurityType.Future} or {SecurityType.Option}."); } if (symbols.Any(x => x.SecurityType != _securityType)) { throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type."); } if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1) { throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker."); } var dataType = LeanData.GetDataType(_resolution, _tickType); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var ticker = symbols.First().ID.Symbol; var market = symbols.First().ID.Market; var canonicalSymbol = Symbol.Create(ticker, _securityType, market); var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType); var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType); var historyBySymbol = new Dictionary <Symbol, List <IGrouping <DateTime, BaseData> > >(); var historyBySymbolDailyOrHour = new Dictionary <Symbol, List <BaseData> >(); foreach (var symbol in symbols) { var historyRequest = new HistoryRequest( startTimeUtc, endTimeUtc, dataType, symbol, _resolution, exchangeHours, dataTimeZone, _resolution, true, false, DataNormalizationMode.Raw, _tickType ); var history = brokerage.GetHistory(historyRequest) .Select( x => { x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone); return(x); }) .ToList(); if (_resolution == Resolution.Daily || _resolution == Resolution.Hour) { historyBySymbolDailyOrHour.Add(symbol, history); } else { // group by date in DataTimeZone var historyByDate = history.GroupBy(x => x.Time.Date).ToList(); historyBySymbol.Add(symbol, historyByDate); } } if (_resolution == Resolution.Daily || _resolution == Resolution.Hour) { SaveDailyOrHour(symbols, canonicalSymbol, historyBySymbolDailyOrHour); } else { SaveMinuteOrSecondOrTick(symbols, startTimeUtc, endTimeUtc, canonicalSymbol, historyBySymbol); } }
public void LiveFillForwardEnumeratorDoesNotStall() { var now = DateTime.UtcNow; var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork); var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>(); var fillForwardEnumerator = new LiveFillForwardEnumerator( timeProvider, enqueueableEnumerator, new SecurityExchange(MarketHoursDatabase.FromDataFolder() .ExchangeHoursListing .First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity) .Value .ExchangeHours), Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()), false, now, Resolution.Minute.ToTimeSpan(), TimeZones.NewYork, new DateTime(2020, 5, 21) ); var openingBar = new TradeBar { Open = 0.01m, High = 0.01m, Low = 0.01m, Close = 0.01m, Volume = 1, EndTime = new DateTime(2020, 5, 21, 9, 40, 0), Symbol = Symbols.AAPL }; var secondBar = new TradeBar { Open = 1m, High = 2m, Low = 1m, Close = 2m, Volume = 100, EndTime = new DateTime(2020, 5, 21, 9, 42, 0), Symbol = Symbols.AAPL }; // Enqueue the first point, which will be emitted ASAP. enqueueableEnumerator.Enqueue(openingBar); Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.NotNull(fillForwardEnumerator.Current); Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open); Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime); // Advance the time, we expect a fill-forward bar. timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100)); Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open); Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime); // Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100)); enqueueableEnumerator.Enqueue(secondBar); Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); }
public void AppliesDividendsOnce() { // init algorithm var algorithm = new AlgorithmStub(); algorithm.SetLiveMode(true); var dividend = new Dividend(Symbols.SPY, DateTime.UtcNow, 10m, 100m); var feed = new TestDividendDataFeed(algorithm, dividend); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm, new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)), algorithm.Settings, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.AddSecurities(equities: new List <string> { "SPY" }); algorithm.Securities[Symbols.SPY].Holdings.SetHoldings(100m, 1); algorithm.PostInitialize(); var initializedCash = algorithm.Portfolio.CashBook["USD"].Amount; // init algorithm manager var manager = new AlgorithmManager(true); var job = new LiveNodePacket { UserId = 1, ProjectId = 2, DeployId = $"{nameof(PaperBrokerageTests)}.{nameof(AppliesDividendsOnce)}" }; var results = new LiveTradingResultHandler(); var transactions = new BacktestingTransactionHandler(); var brokerage = new PaperBrokerage(algorithm, job); // initialize results and transactions results.Initialize(job, new EventMessagingHandler(), new Api.Api(), feed, new BrokerageSetupHandler(), transactions); results.SetAlgorithm(algorithm); transactions.Initialize(algorithm, brokerage, results); // run algorithm manager manager.Run(job, algorithm, dataManager, transactions, results, new BacktestingRealTimeHandler(), new AlgorithmManagerTests.NullLeanManager(), new AlgorithmManagerTests.NullAlphaHandler(), new CancellationToken() ); var postDividendCash = algorithm.Portfolio.CashBook["USD"].Amount; Assert.AreEqual(initializedCash + dividend.Distribution, postDividendCash); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _dataProvider = dataProvider; _dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = subscriptionManager.DataFeedSubscriptions; _universeSelection = subscriptionManager.UniverseSelection; // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); IsActive = true; // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security( exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency), _algorithm.Portfolio.CashBook ); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize if (timeTriggered is UserDefinedUniverse) { var userDefined = (UserDefinedUniverse)timeTriggered; userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator); var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
public EquityExchange() : base(MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, null, SecurityType.Equity, TimeZones.NewYork)) { }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { Subscription subscription = null; // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator = null; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}"); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: {config.Symbol.ID}"); // we subscribe using a normalized symbol, without a random GUID, // since the ticker plant will send the coarse data using this symbol var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false); // Will try to pull coarse data from the data folder every 30min, file with today's date. // If lean is started today it will trigger initial coarse universe selection var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, // we adjust time to the previous tradable date time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false) ); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); // aggregates each coarse data point into a single BaseDataCollection var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, normalizedSymbol, true); _customExchange.AddEnumerator(normalizedSymbol, aggregator); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(normalizedSymbol, data => { var coarseData = data as BaseDataCollection; enqueable.Enqueue(new BaseDataCollection(coarseData.Time, config.Symbol, coarseData.Data)); subscription.OnNewDataAvailable(); }); enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider, // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID); Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) => { var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration); var input = _dataQueueHandler.Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable()); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone, subRequest.StartTimeLocal)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var timeProvider = new PredicateTimeProvider(_timeProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var timeProvider = new PredicateTimeProvider(_timeProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider); // send the subscription for the new symbol through to the data queuehandler if (_channelProvider.ShouldStreamSubscription(_job, subscription.Configuration)) { _dataQueueHandler.Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable()); } return(subscription); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager"></param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; Thread threadAlphas = null; //-> Initialize messaging system _systemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) { IsBackground = true, Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; DataManager dataManager = null; var synchronizer = new Synchronizer(); try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager _systemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance _algorithmHandlers.Alphas.Initialize(job, algorithm, _systemHandlers.Notify, _systemHandlers.Api); // Initialize the brokerage IBrokerageFactory factory; brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, (ISecurityInitializerProvider)algorithm); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(_algorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase); _algorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize( algorithm, dataManager, _liveMode, algorithm.Portfolio.CashBook); // Initialize the data feed before we initialize so he can intercept added securities/universes via events _algorithmHandlers.DataFeed.Initialize( algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, _algorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(_algorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, _algorithmHandlers.DataProvider, historyDataCacheProvider, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } } ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { _algorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { _algorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); // alpha handler needs start/end dates to determine sample step sizes _algorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Interpret(e.InnerException, _exceptionInterpreter); message += _exceptionInterpreter.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + _algorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + _algorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + _algorithmHandlers.Alphas.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting _systemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) { IsBackground = true, Name = "Transaction Thread" }; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) { IsBackground = true, Name = "RealTime Thread" }; threadAlphas = new Thread(() => _algorithmHandlers.Alphas.Run()) { IsBackground = true, Name = "Alpha Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, dataManager, synchronizer, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _systemHandlers.LeanManager, _algorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished _systemHandlers.LeanManager.OnAlgorithmEnd(); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(_algorithmHandlers.Transactions, csvTransactionsFileName); } try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) && charts.ContainsKey(benchmarkKey) && charts[benchmarkKey].Series.ContainsKey(benchmarkKey) ) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); //Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ? (algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue : 0; //Add other fixed parameters. banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2")); banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2")); banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2")); banner.Add("Return", netReturn.ToString("P")); banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2")); } } catch (Exception err) { Log.Error(err, "Error generating statistics packet"); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; if (!_liveMode) { var kps = dataPoints / (double)1000 / totalSeconds; _algorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner); } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); _algorithmHandlers.Alphas.Exit(); dataManager?.RemoveAllSubscriptions(); } //Close result handler: _algorithmHandlers.Results.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive) || (_algorithmHandlers.Alphas != null && _algorithmHandlers.Alphas.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (threadAlphas != null && threadAlphas.IsAlive) { threadAlphas.Abort(); } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (_algorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { _algorithm = algorithm; // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } // attach to the message event to relay brokerage specific initialization messages EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message)); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the default brokerage model before initialize algorithm.SetBrokerageModel(_factory.BrokerageModel); //Margin calls are disabled by default in live mode algorithm.Portfolio.MarginCallModel = MarginCallModel.Null; //Set our parameters algorithm.SetParameters(job.Parameters); algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds()); //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); if (liveJob.Brokerage != "PaperBrokerage") { //Zero the CashBook - we'll populate directly from brokerage foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } } } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } // let the world know what we're doing since logging in can take a minute resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage..."); brokerage.Message += brokerageOnMessage; algorithm.Transactions.SetOrderProcessor(transactionHandler); Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError(string.Format("Error connecting to brokerage: {0}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err.Message)); return(false); } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage..."); try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount); algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage..."); try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage..."); try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var supportedSecurityTypes = new HashSet <SecurityType> { SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd, SecurityType.Option, SecurityType.Future }; var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min()); foreach (var holding in holdings) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); // verify existing holding security type if (!supportedSecurityTypes.Contains(holding.Type)) { Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value); AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " + "QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes)); // keep aggregating these errors continue; } if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString()); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); // if we adding option, we have to add the option universe, so we take underlying symbol // Implemented solution is temporary. We will remove this code once it is the idea on how to refactor IAlgorithm is matured. if (holding.Type == SecurityType.Option) { var underlying = holding.Symbol.Underlying.Value; // adding entire option universe to the system var canonicalOption = algorithm.AddSecurity(holding.Type, underlying, minResolution.Value, null, true, 1.0m, false); var universe = algorithm.UniverseManager.Where(x => x.Key == canonicalOption.Symbol).First().Value; // adding current option contract to the system var option = universe.CreateSecurity(holding.Symbol, algorithm, marketHoursDatabase, symbolPropertiesDatabase); algorithm.Securities.Add(holding.Symbol, option); } else if (holding.Type == SecurityType.Future) { var underlying = holding.Symbol.Underlying.Value; // adding entire future universe to the system var canonicalFuture = algorithm.AddSecurity(holding.Type, underlying, minResolution.Value, null, true, 1.0m, false); var universe = algorithm.UniverseManager.Where(x => x.Key == canonicalFuture.Symbol).First().Value; // adding current future contract to the system var future = universe.CreateSecurity(holding.Symbol, algorithm, marketHoursDatabase, symbolPropertiesDatabase); algorithm.Securities.Add(holding.Symbol, future); } else { // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false); } } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message); return(false); } algorithm.PostInitialize(); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return(Errors.Count == 0); }
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper) : this(dataFeed, algorithm, timeKeeper, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder()) { }
/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public override IEnumerable <BaseData> GetHistory(HistoryRequest request) { if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol)) { Log.Trace("FxcmBrokerage.GetHistory(): Invalid symbol: {0}, no history returned", request.Symbol.Value); yield break; } // cache exchange time zone for symbol DateTimeZone exchangeTimeZone; if (!_symbolExchangeTimeZones.TryGetValue(request.Symbol, out exchangeTimeZone)) { exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.FXCM, request.Symbol, request.Symbol.SecurityType).TimeZone; _symbolExchangeTimeZones.Add(request.Symbol, exchangeTimeZone); } var interval = ToFxcmInterval(request.Resolution); // download data var history = new List <BaseData>(); var lastEndTime = DateTime.MinValue; var end = request.EndTimeUtc; var attempt = 1; while (end > request.StartTimeUtc) { Log.Debug(string.Format("FxcmBrokerage.GetHistory(): Requesting {0:O} to {1:O}", end, request.StartTimeUtc)); _lastHistoryChunk.Clear(); var mdr = new MarketDataRequest(); mdr.setSubscriptionRequestType(SubscriptionRequestTypeFactory.SNAPSHOT); mdr.setResponseFormat(IFixMsgTypeDefs.__Fields.MSGTYPE_FXCMRESPONSE); mdr.setFXCMTimingInterval(interval); mdr.setMDEntryTypeSet(MarketDataRequest.MDENTRYTYPESET_ALL); mdr.setFXCMStartDate(new UTCDate(ToJavaDateUtc(request.StartTimeUtc))); mdr.setFXCMStartTime(new UTCTimeOnly(ToJavaDateUtc(request.StartTimeUtc))); mdr.setFXCMEndDate(new UTCDate(ToJavaDateUtc(end))); mdr.setFXCMEndTime(new UTCTimeOnly(ToJavaDateUtc(end))); mdr.addRelatedSymbol(_fxcmInstruments[_symbolMapper.GetBrokerageSymbol(request.Symbol)]); AutoResetEvent autoResetEvent; lock (_locker) { _currentRequest = _gateway.sendMessage(mdr); autoResetEvent = new AutoResetEvent(false); _mapRequestsToAutoResetEvents[_currentRequest] = autoResetEvent; _pendingHistoryRequests.Add(_currentRequest); } if (!autoResetEvent.WaitOne(HistoryResponseTimeout)) { // No response can mean genuine timeout or the history data has ended. // 90% of the time no response because no data; widen the search net to 5m if we don't get a response: if (request.StartTimeUtc.AddSeconds(300) >= end) { break; } // 5% of the time its because the data ends at a specific, repeatible time not close to our desired endtime: if (end == lastEndTime) { Log.Trace("FxcmBrokerage.GetHistory(): Request for {0} ended at {1:O}", request.Symbol.Value, end); break; } // 5% of the time its because of an internet / time of day / api settings / timeout: throw if this is the *second* attempt. if (EnableOnlyHistoryRequests && lastEndTime != DateTime.MinValue) { throw new TimeoutException(string.Format("FxcmBrokerage.GetHistory(): History operation ending in {0:O} took longer than {1} seconds. This may be because there is no data, retrying...", end, (decimal)HistoryResponseTimeout / 1000)); } // Assuming Timeout: If we've already retried quite a few times, lets bail. if (++attempt > MaximumHistoryRetryAttempts) { Log.Trace("FxcmBrokerage.GetHistory(): Maximum attempts reached for: " + request.Symbol.Value); break; } // Assuming Timeout: Save end time and if have the same endtime next time, break since its likely there's no data after that time. lastEndTime = end; Log.Trace("FxcmBrokerage.GetHistory(): Attempt " + attempt + " for: " + request.Symbol.Value + " ended at " + lastEndTime.ToString("O")); continue; } // Add data lock (_locker) { history.InsertRange(0, _lastHistoryChunk); } var firstDateUtc = _lastHistoryChunk[0].Time.ConvertToUtc(exchangeTimeZone); if (end != firstDateUtc) { // new end date = first datapoint date. end = request.Resolution == Resolution.Tick ? firstDateUtc.AddMilliseconds(-1) : firstDateUtc.AddSeconds(-1); if (request.StartTimeUtc.AddSeconds(10) >= end) { break; } } else { break; } } foreach (var data in history) { yield return(data); } }
public void RefreshesFutureChainUniverseOnDateChange() { var startTime = new DateTime(2018, 10, 17, 10, 0, 0); var timeProvider = new ManualTimeProvider(startTime); var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); var factory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider); var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CBOE, "/VX"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CBOE, canonicalSymbol, SecurityType.Future); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.Chicago, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var future = new Future( canonicalSymbol, exchangeHours, quoteCurrency, SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero); var universe = new FuturesChainUniverse(future, universeSettings); var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime); var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider()); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); var data = enumerator.Current as FuturesChainUniverseDataCollection; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); timeProvider.Advance(Time.OneSecond); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); timeProvider.Advance(Time.OneMinute); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); timeProvider.Advance(Time.OneDay); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current as FuturesChainUniverseDataCollection; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); timeProvider.Advance(Time.OneMinute); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); enumerator.Dispose(); }
private static DateRules GetDateRules() { var timeKeeper = new TimeKeeper(_utcNow, new List <DateTimeZone>()); var manager = new SecurityManager(timeKeeper); // Add SPY for Equity testing var securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, null, SecurityType.Equity); var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false); manager.Add( Symbols.SPY, new Security( securityExchangeHours, config, new Cash(Currencies.USD, 0, 1m), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ) ); // Add BTC for Crypto testing securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.Bitfinex, Symbols.BTCUSD, SecurityType.Crypto); config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.BTCUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false); manager.Add( Symbols.BTCUSD, new Security( securityExchangeHours, config, new Cash(Currencies.USD, 0, 1m), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ) ); // Add EURUSD for Forex testing securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.FXCM, Symbols.EURUSD, SecurityType.Forex); config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.EURUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false); manager.Add( Symbols.EURUSD, new Security( securityExchangeHours, config, new Cash(Currencies.USD, 0, 1m), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ) ); // Add Fut_SPY_Feb19_2016 for testing securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CME, Symbols.Fut_SPY_Feb19_2016, SecurityType.Future); config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.Fut_SPY_Feb19_2016, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false); manager.Add( Symbols.Fut_SPY_Feb19_2016, new Security( securityExchangeHours, config, new Cash(Currencies.USD, 0, 1m), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ) ); var rules = new DateRules(manager, TimeZones.NewYork); return(rules); }
public void ForexCashFills() { // this test asserts the portfolio behaves according to the Test_Cash algo, but for a Forex security, // see TestData\CashTestingStrategy.csv; also "https://www.dropbox.com/s/oiliumoyqqj1ovl/2013-cash.csv?dl=1" const string fillsFile = "TestData\\test_forex_fills.xml"; const string equityFile = "TestData\\test_forex_equity.xml"; const string mchQuantityFile = "TestData\\test_forex_fills_mch_quantity.xml"; const string jwbQuantityFile = "TestData\\test_forex_fills_jwb_quantity.xml"; var fills = XDocument.Load(fillsFile).Descendants("OrderEvent").Select(x => new OrderEvent( x.Get <int>("OrderId"), SymbolMap[x.Get <string>("Symbol")], DateTime.MinValue, x.Get <OrderStatus>("Status"), x.Get <int>("FillQuantity") < 0 ? OrderDirection.Sell : x.Get <int>("FillQuantity") > 0 ? OrderDirection.Buy : OrderDirection.Hold, x.Get <decimal>("FillPrice"), x.Get <int>("FillQuantity"), 0) ).ToList(); var equity = XDocument.Load(equityFile).Descendants("decimal") .Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture)) .ToList(); var mchQuantity = XDocument.Load(mchQuantityFile).Descendants("decimal") .Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture)) .ToList(); var jwbQuantity = XDocument.Load(jwbQuantityFile).Descendants("decimal") .Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture)) .ToList(); Assert.AreEqual(fills.Count + 1, equity.Count); // we're going to process fills and very our equity after each fill var subscriptions = new SubscriptionManager(TimeKeeper); var securities = new SecurityManager(TimeKeeper); var transactions = new SecurityTransactionManager(securities); var portfolio = new SecurityPortfolioManager(securities, transactions); portfolio.SetCash(equity[0]); portfolio.CashBook.Add("MCH", mchQuantity[0], 0); portfolio.CashBook.Add("JWB", jwbQuantity[0], 0); var jwbCash = portfolio.CashBook["JWB"]; var mchCash = portfolio.CashBook["MCH"]; var usdCash = portfolio.CashBook["USD"]; var mchJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, jwbCash, subscriptions.Add(MCHJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(jwbCash.Symbol)); mchJwbSecurity.SetLeverage(10m); var mchUsdSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, usdCash, subscriptions.Add(MCHUSD, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(usdCash.Symbol)); mchUsdSecurity.SetLeverage(10m); var usdJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, mchCash, subscriptions.Add(USDJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(mchCash.Symbol)); usdJwbSecurity.SetLeverage(10m); // no fee model mchJwbSecurity.TransactionModel = new SecurityTransactionModel(); mchUsdSecurity.TransactionModel = new SecurityTransactionModel(); usdJwbSecurity.TransactionModel = new SecurityTransactionModel(); securities.Add(mchJwbSecurity); securities.Add(usdJwbSecurity); securities.Add(mchUsdSecurity); portfolio.CashBook.EnsureCurrencyDataFeeds(securities, subscriptions, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), DefaultBrokerageModel.DefaultMarketMap); for (int i = 0; i < fills.Count; i++) { // before processing the fill we must deduct the cost var fill = fills[i]; var time = DateTime.Today.AddDays(i); // the value of 'MCJWB' increments for each fill, the original test algo did this monthly // the time doesn't really matter though decimal mchJwb = i + 1; decimal mchUsd = (i + 1) / (i + 2m); decimal usdJwb = i + 2; Assert.AreEqual((double)mchJwb, (double)(mchUsd * usdJwb), 1e-10); //Console.WriteLine("Step: " + i + " -- MCHJWB: " + mchJwb); jwbCash.Update(new IndicatorDataPoint(MCHJWB, time, mchJwb)); usdCash.Update(new IndicatorDataPoint(MCHUSD, time, mchUsd)); mchCash.Update(new IndicatorDataPoint(JWBUSD, time, usdJwb)); var updateData = new Dictionary <Security, BaseData> { { mchJwbSecurity, new IndicatorDataPoint(MCHJWB, time, mchJwb) }, { mchUsdSecurity, new IndicatorDataPoint(MCHUSD, time, mchUsd) }, { usdJwbSecurity, new IndicatorDataPoint(JWBUSD, time, usdJwb) } }; foreach (var kvp in updateData) { kvp.Key.SetMarketPrice(kvp.Value); } portfolio.ProcessFill(fill); //Console.WriteLine("-----------------------"); //Console.WriteLine(fill); //Console.WriteLine("Post step: " + i); //foreach (var cash in portfolio.CashBook) //{ // Console.WriteLine(cash.Value); //} //Console.WriteLine("CashValue: " + portfolio.CashBook.TotalValueInAccountCurrency); Console.WriteLine(i + 1 + " " + portfolio.TotalPortfolioValue.ToString("C")); //Assert.AreEqual((double) equity[i + 1], (double)portfolio.TotalPortfolioValue, 2e-2); Assert.AreEqual((double)mchQuantity[i + 1], (double)portfolio.CashBook["MCH"].Amount); Assert.AreEqual((double)jwbQuantity[i + 1], (double)portfolio.CashBook["JWB"].Amount); //Console.WriteLine(); //Console.WriteLine(); } }