/// <summary>
/// Get the last known price using the history provider.
/// Useful for seeding securities with the correct price
/// </summary>
/// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
/// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
public BaseData GetLastKnownPrice(Security security)
{
if (security.Symbol.IsCanonical() || HistoryProvider == null)
{
return(null);
}
var configs = SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol);
// For speed and memory usage, use Resolution.Minute as the minimum resolution
var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)configs.GetHighestResolution());
var isExtendedMarketHours = configs.IsExtendedMarketHours();
var startTime = _historyRequestFactory.GetStartTimeAlgoTz(security.Symbol, 1, resolution, security.Exchange.Hours);
var endTime = Time.RoundDown(resolution.ToTimeSpan());
// request QuoteBar for Options and Futures
var dataType = typeof(BaseData);
if (security.Type == SecurityType.Option || security.Type == SecurityType.Future)
{
dataType = LeanData.GetDataType(resolution, TickType.Quote);
}
// Get the config with the largest resolution
var subscriptionDataConfig = GetMatchingSubscription(security.Symbol, dataType);
// if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar
if (subscriptionDataConfig != null && subscriptionDataConfig.Resolution == Resolution.Tick)
{
dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType);
subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution);
}
var request = new HistoryRequest(
startTime.ConvertToUtc(_localTimeKeeper.TimeZone),
endTime.ConvertToUtc(_localTimeKeeper.TimeZone),
subscriptionDataConfig == null ? typeof(TradeBar) : subscriptionDataConfig.Type,
security.Symbol,
resolution,
security.Exchange.Hours,
MarketHoursDatabase.FromDataFolder().GetDataTimeZone(security.Symbol.ID.Market, security.Symbol, security.Symbol.SecurityType),
resolution,
isExtendedMarketHours,
configs.IsCustomData(),
configs.DataNormalizationMode(),
subscriptionDataConfig == null ? LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), security.Type) : subscriptionDataConfig.TickType
);
var history = History(new List <HistoryRequest> {
request
}).ToList();
if (history.Any() && history.First().Values.Any())
{
return(history.First().Values.First());
}
return(null);
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is ITimeTriggeredUniverse)
{
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_algorithm.UtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e),
_mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider));
}
if (request.Universe is FuturesChainUniverse)
{
return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
return(_subscriptionFactory);
}
public void InitializesFromDataFolder()
{
var provider = MarketHoursDatabase.FromDataFolder();
Assert.AreNotEqual(0, provider.ExchangeHoursListing.Count);
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataProvider = dataProvider;
_dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider);
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
var timeSliceFactory = new TimeSliceFactory(algorithm.TimeZone);
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection);
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
synchronizer.SetTimeSliceFactory(timeSliceFactory);
var totalDataPoints = 0;
var subscriptions = dataManager.DataFeedSubscriptions;
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// log what we're doing
Log.Trace($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime = DateTime.MaxValue;
DateTime previousValue;
var stopwatch = Stopwatch.StartNew();
var enumerator = synchronizer.Sync(subscriptions, CancellationToken.None).GetEnumerator();
do
{
previousValue = currentTime;
enumerator.MoveNext();
var timeSlice = enumerator.Current;
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != previousValue);
stopwatch.Stop();
enumerator.DisposeSafely();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Log.Trace($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
dataManager.RemoveAllSubscriptions();
}
public void TestBasicFeaturesWithOptionsFutures()
{
var securities = new SecurityManager(TimeKeeper);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
securities.Add(
Symbols.SPY,
new Security(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Equity, Symbols.SPY),
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
securities[Symbols.SPY].SetMarketPrice(new TradeBar {
Time = securities.UtcTime, Symbol = Symbols.SPY, Close = 195
});
var option1 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 192m, new DateTime(2016, 02, 16));
securities.Add(
option1,
new Option(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Option, option1),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var option2 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 193m, new DateTime(2016, 03, 19));
securities.Add(
option2,
new Option(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Option, option2),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var future1 = Symbol.CreateFuture("ES", Market.USA, new DateTime(2016, 02, 16));
securities.Add(
future1,
new Future(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Future, future1),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var future2 = Symbol.CreateFuture("ES", Market.USA, new DateTime(2016, 02, 19));
securities.Add(
future2,
new Future(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Future, future2),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var cal = new TradingCalendar(securities, marketHoursDatabase);
var optionDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count();
Assert.AreEqual(2, optionDays);
var futureDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count();
Assert.AreEqual(2, futureDays);
var days = cal.GetTradingDays(new DateTime(2016, 02, 16), new DateTime(2016, 03, 19));
var optionAndfutureDays = days.Where(x => x.FutureExpirations.Any() || x.OptionExpirations.Any()).Count();
Assert.AreEqual(3, optionAndfutureDays);
// why? because option1 and future1 expire in one day 2016-02-16. Lets have a look.
var day = cal.GetTradingDay(new DateTime(2016, 02, 16));
Assert.AreEqual(1, day.OptionExpirations.Count());
Assert.AreEqual(1, day.FutureExpirations.Count());
var businessDays = days.Where(x => x.BusinessDay).Count();
Assert.AreEqual(24, businessDays);
var weekends = days.Where(x => x.Weekend).Count();
Assert.AreEqual(9, weekends);
Assert.AreEqual(24 + 9, (new DateTime(2016, 03, 19) - new DateTime(2016, 02, 16)).TotalDays + 1 /*inclusive*/);
}
public void RefreshesOptionChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 19, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache(),
null
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
EnqueueableEnumerator <BaseData> underlyingEnumerator = null;
Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req) =>
{
underlyingEnumerator = new EnqueueableEnumerator <BaseData>();
return(new LiveFillForwardEnumerator(
timeProvider,
underlyingEnumerator,
option.Exchange,
fillForwardResolution,
false,
Time.EndOfTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new OptionChainUniverse(option, universeSettings, true);
var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, TestGlobals.DataProvider);
// 2018-10-19 10:00 AM UTC
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
// 2018-10-19 10:01 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:02 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:03 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:03 AM UTC
timeProvider.Advance(Time.OneDay);
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:04 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.Enqueue(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
enumerator.Dispose();
}
/// <summary>
/// Creates an instance of the PortfolioLooper class
/// </summary>
/// <param name="startingCash">Equity curve</param>
/// <param name="orders">Order events</param>
/// <param name="resolution">Optional parameter to override default resolution (Hourly)</param>
private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution)
{
// Initialize the providers that the HistoryProvider requires
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider");
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider");
var dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false);
var historyProvider = Composer.Instance.GetExportedValueByTypeName <IHistoryProvider>("SubscriptionDataReaderHistoryProvider");
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, dataCacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false));
Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders);
Algorithm.SetHistoryProvider(historyProvider);
// Dummy LEAN datafeed classes and initializations that essentially do nothing
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"");
var feed = new MockDataFeed();
// Create MHDB and Symbol properties DB instances for the DataManager
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
_dataManager = new DataManager(feed,
new UniverseSelection(
Algorithm,
new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio))),
Algorithm,
Algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null);
_securityService = new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio));
var transactions = new BacktestingTransactionHandler();
var results = new BacktestingResultHandler();
// Initialize security services and other properties so that we
// don't get null reference exceptions during our re-calculation
Algorithm.Securities.SetSecurityService(_securityService);
Algorithm.SubscriptionManager.SetDataManager(_dataManager);
// Initializes all the proper Securities from the orders provided by the user
Algorithm.FromOrders(orders);
// Initialize the algorithm
Algorithm.Initialize();
Algorithm.PostInitialize();
// More initialization, this time with Algorithm and other misc. classes
results.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions);
results.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), results);
feed.Initialize(Algorithm, job, results, null, null, null, _dataManager, null);
// Begin setting up the currency conversion feed if needed
var coreSecurities = Algorithm.Securities.Values.ToList();
if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto))
{
BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection);
var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList();
// Skip the history request if we don't need to convert anything
if (conversionSecurities.Any())
{
// Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency
_conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution);
}
}
}
public void DoesNotEmitInvalidData()
{
var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
var endTime = new DateTime(2014, 06, 09, 20, 0, 0);
var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
return(new FillForwardEnumerator(
input,
option.Exchange,
fillForwardResolution,
false,
endTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
startTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc,
MapFileResolver.Create(Globals.DataFolder, Market.USA),
new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()));
var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
var emittedCount = 0;
foreach (var data in enumerator.AsEnumerable())
{
emittedCount++;
var optionData = data as OptionChainUniverseDataCollection;
Assert.IsNotNull(optionData);
Assert.IsNotNull(optionData.Underlying);
Assert.AreNotEqual(0, optionData.Data.Count);
}
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, emittedCount);
}
/// <summary>
/// Initializes a new instance of the <see cref="CryptoExchange"/> class using market hours
/// derived from the market-hours-database for the Crypto market
/// </summary>
public CryptoExchange(string market)
: base(MarketHoursDatabase.FromDataFolder().GetExchangeHours(market, null, SecurityType.Crypto, TimeZones.Utc))
{
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
var feed = new AlgorithmManagerTests.MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm);
algorithm.Securities.SetSecurityService(securityService);
var dataManager = new DataManager(feed,
new UniverseSelection(feed, algorithm, securityService),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection, algorithm.Portfolio.CashBook);
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
synchronizer.SetSliceTimeZone(algorithm.TimeZone);
var totalDataPoints = 0;
var subscriptions = dataManager.DataFeedSubscriptions;
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// force JIT
synchronizer.Sync(subscriptions);
// log what we're doing
Console.WriteLine($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime = DateTime.MaxValue;
DateTime previousValue;
var stopwatch = Stopwatch.StartNew();
do
{
previousValue = currentTime;
var timeSlice = synchronizer.Sync(subscriptions);
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != previousValue);
stopwatch.Stop();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Downloads historical data from the brokerage and saves it in LEAN format.
/// </summary>
/// <param name="brokerage">The brokerage from where to fetch the data</param>
/// <param name="symbols">The list of symbols</param>
/// <param name="startTimeUtc">The starting date/time (UTC)</param>
/// <param name="endTimeUtc">The ending date/time (UTC)</param>
public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc)
{
if (symbols.Count == 0)
{
throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty.");
}
if (_tickType != TickType.Trade && _tickType != TickType.Quote)
{
throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote.");
}
if (symbols.Any(x => x.SecurityType != _securityType))
{
throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type.");
}
if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1)
{
throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker.");
}
var dataType = LeanData.GetDataType(_resolution, _tickType);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var ticker = symbols.First().ID.Symbol;
var market = symbols.First().ID.Market;
var canonicalSymbol = Symbol.Create(ticker, _securityType, market);
var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
foreach (var symbol in symbols)
{
var historyRequest = new HistoryRequest(
startTimeUtc,
endTimeUtc,
dataType,
symbol,
_resolution,
exchangeHours,
dataTimeZone,
_resolution,
true,
false,
DataNormalizationMode.Raw,
_tickType
);
var history = brokerage.GetHistory(historyRequest)
.Select(
x =>
{
// Convert to date timezone before we write it
x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
return(x);
})
.ToList();
// Generate a writer for this data and write it
var writer = new LeanDataWriter(_resolution, symbol, _dataDirectory, _tickType);
writer.Write(history);
}
}
public void FastExitsDoNotThrowUnhandledExceptions()
{
var algorithm = new AlgorithmStub();
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var feed = new TestableLiveTradingDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm);
algorithm.Securities.SetSecurityService(securityService);
var dataManager = new DataManager(feed,
new UniverseSelection(algorithm, securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new TestableSynchronizer(_algorithm, dataManager, true);
algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick {
Symbol = SymbolCache.GetSymbol(x.ToString())
}).ToList();
feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
feed.Initialize(
algorithm,
job,
resultHandler,
mapFileProvider,
new LocalDiskFactorFileProvider(mapFileProvider),
fileProvider,
dataManager,
synchronizer);
var unhandledExceptionWasThrown = false;
try
{
feed.Exit();
}
catch (Exception ex)
{
QuantConnect.Logging.Log.Error(ex.ToString());
unhandledExceptionWasThrown = true;
}
Thread.Sleep(500);
Assert.IsFalse(unhandledExceptionWasThrown);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return(subscription);
}
/// <summary>
/// Downloads historical data from the brokerage and saves it in LEAN format.
/// </summary>
/// <param name="brokerage">The brokerage from where to fetch the data</param>
/// <param name="symbols">The list of symbols</param>
/// <param name="startTimeUtc">The starting date/time (UTC)</param>
/// <param name="endTimeUtc">The ending date/time (UTC)</param>
public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc)
{
if (symbols.Count == 0)
{
throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty.");
}
if (_tickType != TickType.Trade && _tickType != TickType.Quote)
{
throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote.");
}
if (_securityType != SecurityType.Future && _securityType != SecurityType.Option && _securityType != SecurityType.FutureOption)
{
throw new ArgumentException($"DownloadAndSave(): The security type must be {SecurityType.Future} or {SecurityType.Option}.");
}
if (symbols.Any(x => x.SecurityType != _securityType))
{
throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type.");
}
if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1)
{
throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker.");
}
var dataType = LeanData.GetDataType(_resolution, _tickType);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var ticker = symbols.First().ID.Symbol;
var market = symbols.First().ID.Market;
var canonicalSymbol = Symbol.Create(ticker, _securityType, market);
var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var historyBySymbol = new Dictionary <Symbol, List <IGrouping <DateTime, BaseData> > >();
var historyBySymbolDailyOrHour = new Dictionary <Symbol, List <BaseData> >();
foreach (var symbol in symbols)
{
var historyRequest = new HistoryRequest(
startTimeUtc,
endTimeUtc,
dataType,
symbol,
_resolution,
exchangeHours,
dataTimeZone,
_resolution,
true,
false,
DataNormalizationMode.Raw,
_tickType
);
var history = brokerage.GetHistory(historyRequest)
.Select(
x =>
{
x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
return(x);
})
.ToList();
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
historyBySymbolDailyOrHour.Add(symbol, history);
}
else
{
// group by date in DataTimeZone
var historyByDate = history.GroupBy(x => x.Time.Date).ToList();
historyBySymbol.Add(symbol, historyByDate);
}
}
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
SaveDailyOrHour(symbols, canonicalSymbol, historyBySymbolDailyOrHour);
}
else
{
SaveMinuteOrSecondOrTick(symbols, startTimeUtc, endTimeUtc, canonicalSymbol, historyBySymbol);
}
}
public void LiveFillForwardEnumeratorDoesNotStall()
{
var now = DateTime.UtcNow;
var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork);
var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>();
var fillForwardEnumerator = new LiveFillForwardEnumerator(
timeProvider,
enqueueableEnumerator,
new SecurityExchange(MarketHoursDatabase.FromDataFolder()
.ExchangeHoursListing
.First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
.Value
.ExchangeHours),
Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()),
false,
now,
Resolution.Minute.ToTimeSpan(),
TimeZones.NewYork,
new DateTime(2020, 5, 21)
);
var openingBar = new TradeBar
{
Open = 0.01m,
High = 0.01m,
Low = 0.01m,
Close = 0.01m,
Volume = 1,
EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
Symbol = Symbols.AAPL
};
var secondBar = new TradeBar
{
Open = 1m,
High = 2m,
Low = 1m,
Close = 2m,
Volume = 100,
EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
Symbol = Symbols.AAPL
};
// Enqueue the first point, which will be emitted ASAP.
enqueueableEnumerator.Enqueue(openingBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.NotNull(fillForwardEnumerator.Current);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
// Advance the time, we expect a fill-forward bar.
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100));
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
enqueueableEnumerator.Enqueue(secondBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
}
public void AppliesDividendsOnce()
{
// init algorithm
var algorithm = new AlgorithmStub();
algorithm.SetLiveMode(true);
var dividend = new Dividend(Symbols.SPY, DateTime.UtcNow, 10m, 100m);
var feed = new TestDividendDataFeed(algorithm, dividend);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataManager = new DataManager(feed,
new UniverseSelection(feed,
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(equities: new List <string> {
"SPY"
});
algorithm.Securities[Symbols.SPY].Holdings.SetHoldings(100m, 1);
algorithm.PostInitialize();
var initializedCash = algorithm.Portfolio.CashBook["USD"].Amount;
// init algorithm manager
var manager = new AlgorithmManager(true);
var job = new LiveNodePacket
{
UserId = 1,
ProjectId = 2,
DeployId = $"{nameof(PaperBrokerageTests)}.{nameof(AppliesDividendsOnce)}"
};
var results = new LiveTradingResultHandler();
var transactions = new BacktestingTransactionHandler();
var brokerage = new PaperBrokerage(algorithm, job);
// initialize results and transactions
results.Initialize(job, new EventMessagingHandler(), new Api.Api(), feed, new BrokerageSetupHandler(), transactions);
results.SetAlgorithm(algorithm);
transactions.Initialize(algorithm, brokerage, results);
// run algorithm manager
manager.Run(job,
algorithm,
dataManager,
transactions,
results,
new BacktestingRealTimeHandler(),
new AlgorithmManagerTests.NullLeanManager(),
new AlgorithmManagerTests.NullAlphaHandler(),
new CancellationToken()
);
var postDividendCash = algorithm.Portfolio.CashBook["USD"].Amount;
Assert.AreEqual(initializedCash + dividend.Distribution, postDividendCash);
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler,
IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider,
IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataProvider = dataProvider;
_dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider);
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_universeSelection = subscriptionManager.UniverseSelection;
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
IsActive = true;
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(
exchangeHours,
config,
_algorithm.Portfolio.CashBook[CashBook.AccountCurrency],
SymbolProperties.GetDefault(CashBook.AccountCurrency),
_algorithm.Portfolio.CashBook
);
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator);
var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
public EquityExchange()
: base(MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, null, SecurityType.Equity, TimeZones.NewYork))
{
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: {config.Symbol.ID}");
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// Will try to pull coarse data from the data folder every 30min, file with today's date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, normalizedSymbol, true);
_customExchange.AddEnumerator(normalizedSymbol, aggregator);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(normalizedSymbol, data =>
{
var coarseData = data as BaseDataCollection;
enqueable.Enqueue(new BaseDataCollection(coarseData.Time, config.Symbol, coarseData.Data));
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = _dataQueueHandler.Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone, subRequest.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(_job, subscription.Configuration))
{
_dataQueueHandler.Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
}
return(subscription);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager"></param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
Thread threadAlphas = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0)
{
IsBackground = true, Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
DataManager dataManager = null;
var synchronizer = new Synchronizer();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
_systemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
_algorithmHandlers.Alphas.Initialize(job, algorithm, _systemHandlers.Notify, _systemHandlers.Api);
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
(ISecurityInitializerProvider)algorithm);
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(_algorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase);
_algorithmHandlers.Results.SetDataManager(dataManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(
algorithm,
dataManager,
_liveMode,
algorithm.Portfolio.CashBook);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
_algorithmHandlers.DataFeed.Initialize(
algorithm,
job,
_algorithmHandlers.Results,
_algorithmHandlers.MapFileProvider,
_algorithmHandlers.FactorFileProvider,
_algorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(_algorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
_algorithmHandlers.DataProvider,
historyDataCacheProvider,
_algorithmHandlers.MapFileProvider,
_algorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
}
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { _algorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { _algorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
// alpha handler needs start/end dates to determine sample step sizes
_algorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Interpret(e.InnerException, _exceptionInterpreter);
message += _exceptionInterpreter.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + _algorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + _algorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + _algorithmHandlers.Alphas.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
_systemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run)
{
IsBackground = true, Name = "Transaction Thread"
};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run)
{
IsBackground = true, Name = "RealTime Thread"
};
threadAlphas = new Thread(() => _algorithmHandlers.Alphas.Run())
{
IsBackground = true, Name = "Alpha Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, dataManager, synchronizer, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _systemHandlers.LeanManager, _algorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
_systemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(_algorithmHandlers.Transactions, csvTransactionsFileName);
}
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) &&
charts.ContainsKey(benchmarkKey) &&
charts[benchmarkKey].Series.ContainsKey(benchmarkKey)
)
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ?
(algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue
: 0;
//Add other fixed parameters.
banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2"));
banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2"));
banner.Add("Return", netReturn.ToString("P"));
banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
}
}
catch (Exception err)
{
Log.Error(err, "Error generating statistics packet");
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
if (!_liveMode)
{
var kps = dataPoints / (double)1000 / totalSeconds;
_algorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
_algorithmHandlers.Alphas.Exit();
dataManager?.RemoveAllSubscriptions();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive ||
(_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) ||
(_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) ||
(_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive) ||
(_algorithmHandlers.Alphas != null && _algorithmHandlers.Alphas.IsActive)) &&
ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (threadAlphas != null && threadAlphas.IsAlive)
{
threadAlphas.Abort();
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (_algorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Margin calls are disabled by default in live mode
algorithm.Portfolio.MarginCallModel = MarginCallModel.Null;
//Set our parameters
algorithm.SetParameters(job.Parameters);
algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds());
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet <SecurityType> {
SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd, SecurityType.Option, SecurityType.Future
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString());
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
// if we adding option, we have to add the option universe, so we take underlying symbol
// Implemented solution is temporary. We will remove this code once it is the idea on how to refactor IAlgorithm is matured.
if (holding.Type == SecurityType.Option)
{
var underlying = holding.Symbol.Underlying.Value;
// adding entire option universe to the system
var canonicalOption = algorithm.AddSecurity(holding.Type, underlying, minResolution.Value, null, true, 1.0m, false);
var universe = algorithm.UniverseManager.Where(x => x.Key == canonicalOption.Symbol).First().Value;
// adding current option contract to the system
var option = universe.CreateSecurity(holding.Symbol, algorithm, marketHoursDatabase, symbolPropertiesDatabase);
algorithm.Securities.Add(holding.Symbol, option);
}
else if (holding.Type == SecurityType.Future)
{
var underlying = holding.Symbol.Underlying.Value;
// adding entire future universe to the system
var canonicalFuture = algorithm.AddSecurity(holding.Type, underlying, minResolution.Value, null, true, 1.0m, false);
var universe = algorithm.UniverseManager.Where(x => x.Key == canonicalFuture.Symbol).First().Value;
// adding current future contract to the system
var future = universe.CreateSecurity(holding.Symbol, algorithm, marketHoursDatabase, symbolPropertiesDatabase);
algorithm.Securities.Add(holding.Symbol, future);
}
else
{
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false);
}
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return(false);
}
algorithm.PostInitialize();
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper)
: this(dataFeed, algorithm, timeKeeper, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder())
{
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(HistoryRequest request)
{
if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol))
{
Log.Trace("FxcmBrokerage.GetHistory(): Invalid symbol: {0}, no history returned", request.Symbol.Value);
yield break;
}
// cache exchange time zone for symbol
DateTimeZone exchangeTimeZone;
if (!_symbolExchangeTimeZones.TryGetValue(request.Symbol, out exchangeTimeZone))
{
exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.FXCM, request.Symbol, request.Symbol.SecurityType).TimeZone;
_symbolExchangeTimeZones.Add(request.Symbol, exchangeTimeZone);
}
var interval = ToFxcmInterval(request.Resolution);
// download data
var history = new List <BaseData>();
var lastEndTime = DateTime.MinValue;
var end = request.EndTimeUtc;
var attempt = 1;
while (end > request.StartTimeUtc)
{
Log.Debug(string.Format("FxcmBrokerage.GetHistory(): Requesting {0:O} to {1:O}", end, request.StartTimeUtc));
_lastHistoryChunk.Clear();
var mdr = new MarketDataRequest();
mdr.setSubscriptionRequestType(SubscriptionRequestTypeFactory.SNAPSHOT);
mdr.setResponseFormat(IFixMsgTypeDefs.__Fields.MSGTYPE_FXCMRESPONSE);
mdr.setFXCMTimingInterval(interval);
mdr.setMDEntryTypeSet(MarketDataRequest.MDENTRYTYPESET_ALL);
mdr.setFXCMStartDate(new UTCDate(ToJavaDateUtc(request.StartTimeUtc)));
mdr.setFXCMStartTime(new UTCTimeOnly(ToJavaDateUtc(request.StartTimeUtc)));
mdr.setFXCMEndDate(new UTCDate(ToJavaDateUtc(end)));
mdr.setFXCMEndTime(new UTCTimeOnly(ToJavaDateUtc(end)));
mdr.addRelatedSymbol(_fxcmInstruments[_symbolMapper.GetBrokerageSymbol(request.Symbol)]);
AutoResetEvent autoResetEvent;
lock (_locker)
{
_currentRequest = _gateway.sendMessage(mdr);
autoResetEvent = new AutoResetEvent(false);
_mapRequestsToAutoResetEvents[_currentRequest] = autoResetEvent;
_pendingHistoryRequests.Add(_currentRequest);
}
if (!autoResetEvent.WaitOne(HistoryResponseTimeout))
{
// No response can mean genuine timeout or the history data has ended.
// 90% of the time no response because no data; widen the search net to 5m if we don't get a response:
if (request.StartTimeUtc.AddSeconds(300) >= end)
{
break;
}
// 5% of the time its because the data ends at a specific, repeatible time not close to our desired endtime:
if (end == lastEndTime)
{
Log.Trace("FxcmBrokerage.GetHistory(): Request for {0} ended at {1:O}", request.Symbol.Value, end);
break;
}
// 5% of the time its because of an internet / time of day / api settings / timeout: throw if this is the *second* attempt.
if (EnableOnlyHistoryRequests && lastEndTime != DateTime.MinValue)
{
throw new TimeoutException(string.Format("FxcmBrokerage.GetHistory(): History operation ending in {0:O} took longer than {1} seconds. This may be because there is no data, retrying...", end, (decimal)HistoryResponseTimeout / 1000));
}
// Assuming Timeout: If we've already retried quite a few times, lets bail.
if (++attempt > MaximumHistoryRetryAttempts)
{
Log.Trace("FxcmBrokerage.GetHistory(): Maximum attempts reached for: " + request.Symbol.Value);
break;
}
// Assuming Timeout: Save end time and if have the same endtime next time, break since its likely there's no data after that time.
lastEndTime = end;
Log.Trace("FxcmBrokerage.GetHistory(): Attempt " + attempt + " for: " + request.Symbol.Value + " ended at " + lastEndTime.ToString("O"));
continue;
}
// Add data
lock (_locker)
{
history.InsertRange(0, _lastHistoryChunk);
}
var firstDateUtc = _lastHistoryChunk[0].Time.ConvertToUtc(exchangeTimeZone);
if (end != firstDateUtc)
{
// new end date = first datapoint date.
end = request.Resolution == Resolution.Tick ? firstDateUtc.AddMilliseconds(-1) : firstDateUtc.AddSeconds(-1);
if (request.StartTimeUtc.AddSeconds(10) >= end)
{
break;
}
}
else
{
break;
}
}
foreach (var data in history)
{
yield return(data);
}
}
public void RefreshesFutureChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 17, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
var factory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CBOE, "/VX");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CBOE, canonicalSymbol, SecurityType.Future);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.Chicago,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var future = new Future(
canonicalSymbol,
exchangeHours,
quoteCurrency,
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new FuturesChainUniverse(future, universeSettings);
var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current as FuturesChainUniverseDataCollection;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
timeProvider.Advance(Time.OneSecond);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneDay);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current as FuturesChainUniverseDataCollection;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
enumerator.Dispose();
}
private static DateRules GetDateRules()
{
var timeKeeper = new TimeKeeper(_utcNow, new List <DateTimeZone>());
var manager = new SecurityManager(timeKeeper);
// Add SPY for Equity testing
var securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, null, SecurityType.Equity);
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.SPY,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add BTC for Crypto testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.Bitfinex, Symbols.BTCUSD, SecurityType.Crypto);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.BTCUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.BTCUSD,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add EURUSD for Forex testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.FXCM, Symbols.EURUSD, SecurityType.Forex);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.EURUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.EURUSD,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add Fut_SPY_Feb19_2016 for testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CME, Symbols.Fut_SPY_Feb19_2016, SecurityType.Future);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.Fut_SPY_Feb19_2016, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.Fut_SPY_Feb19_2016,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
var rules = new DateRules(manager, TimeZones.NewYork);
return(rules);
}
public void ForexCashFills()
{
// this test asserts the portfolio behaves according to the Test_Cash algo, but for a Forex security,
// see TestData\CashTestingStrategy.csv; also "https://www.dropbox.com/s/oiliumoyqqj1ovl/2013-cash.csv?dl=1"
const string fillsFile = "TestData\\test_forex_fills.xml";
const string equityFile = "TestData\\test_forex_equity.xml";
const string mchQuantityFile = "TestData\\test_forex_fills_mch_quantity.xml";
const string jwbQuantityFile = "TestData\\test_forex_fills_jwb_quantity.xml";
var fills = XDocument.Load(fillsFile).Descendants("OrderEvent").Select(x => new OrderEvent(
x.Get <int>("OrderId"),
SymbolMap[x.Get <string>("Symbol")],
DateTime.MinValue,
x.Get <OrderStatus>("Status"),
x.Get <int>("FillQuantity") < 0 ? OrderDirection.Sell
: x.Get <int>("FillQuantity") > 0 ? OrderDirection.Buy
: OrderDirection.Hold,
x.Get <decimal>("FillPrice"),
x.Get <int>("FillQuantity"),
0)
).ToList();
var equity = XDocument.Load(equityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
var mchQuantity = XDocument.Load(mchQuantityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
var jwbQuantity = XDocument.Load(jwbQuantityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
Assert.AreEqual(fills.Count + 1, equity.Count);
// we're going to process fills and very our equity after each fill
var subscriptions = new SubscriptionManager(TimeKeeper);
var securities = new SecurityManager(TimeKeeper);
var transactions = new SecurityTransactionManager(securities);
var portfolio = new SecurityPortfolioManager(securities, transactions);
portfolio.SetCash(equity[0]);
portfolio.CashBook.Add("MCH", mchQuantity[0], 0);
portfolio.CashBook.Add("JWB", jwbQuantity[0], 0);
var jwbCash = portfolio.CashBook["JWB"];
var mchCash = portfolio.CashBook["MCH"];
var usdCash = portfolio.CashBook["USD"];
var mchJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, jwbCash, subscriptions.Add(MCHJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(jwbCash.Symbol));
mchJwbSecurity.SetLeverage(10m);
var mchUsdSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, usdCash, subscriptions.Add(MCHUSD, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(usdCash.Symbol));
mchUsdSecurity.SetLeverage(10m);
var usdJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, mchCash, subscriptions.Add(USDJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork), SymbolProperties.GetDefault(mchCash.Symbol));
usdJwbSecurity.SetLeverage(10m);
// no fee model
mchJwbSecurity.TransactionModel = new SecurityTransactionModel();
mchUsdSecurity.TransactionModel = new SecurityTransactionModel();
usdJwbSecurity.TransactionModel = new SecurityTransactionModel();
securities.Add(mchJwbSecurity);
securities.Add(usdJwbSecurity);
securities.Add(mchUsdSecurity);
portfolio.CashBook.EnsureCurrencyDataFeeds(securities, subscriptions, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), DefaultBrokerageModel.DefaultMarketMap);
for (int i = 0; i < fills.Count; i++)
{
// before processing the fill we must deduct the cost
var fill = fills[i];
var time = DateTime.Today.AddDays(i);
// the value of 'MCJWB' increments for each fill, the original test algo did this monthly
// the time doesn't really matter though
decimal mchJwb = i + 1;
decimal mchUsd = (i + 1) / (i + 2m);
decimal usdJwb = i + 2;
Assert.AreEqual((double)mchJwb, (double)(mchUsd * usdJwb), 1e-10);
//Console.WriteLine("Step: " + i + " -- MCHJWB: " + mchJwb);
jwbCash.Update(new IndicatorDataPoint(MCHJWB, time, mchJwb));
usdCash.Update(new IndicatorDataPoint(MCHUSD, time, mchUsd));
mchCash.Update(new IndicatorDataPoint(JWBUSD, time, usdJwb));
var updateData = new Dictionary <Security, BaseData>
{
{ mchJwbSecurity, new IndicatorDataPoint(MCHJWB, time, mchJwb) },
{ mchUsdSecurity, new IndicatorDataPoint(MCHUSD, time, mchUsd) },
{ usdJwbSecurity, new IndicatorDataPoint(JWBUSD, time, usdJwb) }
};
foreach (var kvp in updateData)
{
kvp.Key.SetMarketPrice(kvp.Value);
}
portfolio.ProcessFill(fill);
//Console.WriteLine("-----------------------");
//Console.WriteLine(fill);
//Console.WriteLine("Post step: " + i);
//foreach (var cash in portfolio.CashBook)
//{
// Console.WriteLine(cash.Value);
//}
//Console.WriteLine("CashValue: " + portfolio.CashBook.TotalValueInAccountCurrency);
Console.WriteLine(i + 1 + " " + portfolio.TotalPortfolioValue.ToString("C"));
//Assert.AreEqual((double) equity[i + 1], (double)portfolio.TotalPortfolioValue, 2e-2);
Assert.AreEqual((double)mchQuantity[i + 1], (double)portfolio.CashBook["MCH"].Amount);
Assert.AreEqual((double)jwbQuantity[i + 1], (double)portfolio.CashBook["JWB"].Amount);
//Console.WriteLine();
//Console.WriteLine();
}
}