public TradingCalendar(SecurityManager securityManager, MarketHoursDatabase marketHoursDatabase)
{
_securityManager = securityManager;
_marketHoursDatabase = marketHoursDatabase;
}
public IQFeedFileHistoryProvider(LookupClient lookupClient, ISymbolMapper symbolMapper, MarketHoursDatabase marketHoursDatabase)
{
_lookupClient = lookupClient;
_symbolMapper = symbolMapper;
_marketHoursDatabase = marketHoursDatabase;
_filesByRequestKeyCache = new ConcurrentDictionary <string, string>();
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString,
string optionPriceEngineName,
string volatilityModelResolutionString,
string chainSymbolCountString,
List <string> tickers
)
{
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
optionPriceEngineName,
volatilityModelResolutionString,
chainSymbolCountString,
tickers
);
if (settings.Start.Year < 1998)
{
Log.Error($"RandomDataGeneratorProgram(): Required parameter --start must be at least 19980101");
Environment.Exit(1);
}
var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
var securityService = new SecurityService(
new CashBook(),
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
{
// init price
security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
// from settings
security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
// from settings
if (security is Option option)
{
option.PriceModel = OptionPriceModels.Create(settings.OptionPriceEngineName, Statistics.PortfolioStatistics.GetRiskFreeRate());
}
})),
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(
new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager))),
new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider")))
);
securityManager.SetSecurityService(securityService);
var generator = new RandomDataGenerator();
generator.Init(settings, securityManager);
generator.Run();
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
Log.Trace("RandomDataGeneratorProgram(): Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
if (!Console.IsInputRedirected)
{
Log.Trace("RandomDataGeneratorProgram(): Press any key to exit...");
Console.ReadKey();
}
}
public virtual Security CreateSecurity(Symbol symbol, IAlgorithm algorithm, MarketHoursDatabase marketHoursDatabase, SymbolPropertiesDatabase symbolPropertiesDatabase)
{
throw new InvalidOperationException("CreateSecurity is obsolete and should not be called." +
"The system will create the required Securities based on selected symbols");
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = config.Symbol
}).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
var cf = new CoarseFundamental();
// load coarse data day by day
enumerator = (from date in Time.EachTradeableDayInTimeZone(security.Exchange.Hours, _algorithm.StartDate, _algorithm.EndDate, config.DataTimeZone, config.ExtendedMarketHours)
let source = cf.GetSource(config, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), config.Symbol, coarseFundamentalForDate)
).GetEnumerator();
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
ScheduleEnumerator(enumerator, enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else if (config.SecurityType == SecurityType.Option && security is Option)
{
var configs = universe.GetSubscriptions(security);
var enumerators = configs.Select(c =>
CreateSubscriptionEnumerator(security, c, localStartTime, localEndTime, _mapFileProvider.Get(c.Market), tradeableDates, false, true)
).ToList();
var sync = new SynchronizingEnumerator(enumerators);
enumerator = new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, MapFileResolver.Empty, tradeableDates, true, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.TryAdd(subscription);
UpdateFillForwardResolution();
}
public void RefreshesFutureChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 17, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
var factory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CBOE, "/VX");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CBOE, canonicalSymbol, SecurityType.Future);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.Chicago,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var future = new Future(
canonicalSymbol,
exchangeHours,
quoteCurrency,
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new FuturesChainUniverse(future, universeSettings);
var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current as FuturesChainUniverseDataCollection;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
timeProvider.Advance(Time.OneSecond);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneDay);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current as FuturesChainUniverseDataCollection;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
enumerator.Dispose();
}
/// <summary>
/// Downloads historical data from the brokerage and saves it in LEAN format.
/// </summary>
/// <param name="brokerage">The brokerage from where to fetch the data</param>
/// <param name="symbols">The list of symbols</param>
/// <param name="startTimeUtc">The starting date/time (UTC)</param>
/// <param name="endTimeUtc">The ending date/time (UTC)</param>
public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc)
{
if (symbols.Count == 0)
{
throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty.");
}
if (_tickType != TickType.Trade && _tickType != TickType.Quote)
{
throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote.");
}
if (_securityType != SecurityType.Future && _securityType != SecurityType.Option && _securityType != SecurityType.FutureOption)
{
throw new ArgumentException($"DownloadAndSave(): The security type must be {SecurityType.Future} or {SecurityType.Option}.");
}
if (symbols.Any(x => x.SecurityType != _securityType))
{
throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type.");
}
if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1)
{
throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker.");
}
var dataType = LeanData.GetDataType(_resolution, _tickType);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var ticker = symbols.First().ID.Symbol;
var market = symbols.First().ID.Market;
var canonicalSymbol = Symbol.Create(ticker, _securityType, market);
var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var historyBySymbol = new Dictionary <Symbol, List <IGrouping <DateTime, BaseData> > >();
var historyBySymbolDailyOrHour = new Dictionary <Symbol, List <BaseData> >();
foreach (var symbol in symbols)
{
var historyRequest = new HistoryRequest(
startTimeUtc,
endTimeUtc,
dataType,
symbol,
_resolution,
exchangeHours,
dataTimeZone,
_resolution,
true,
false,
DataNormalizationMode.Raw,
_tickType
);
var history = brokerage.GetHistory(historyRequest)
.Select(
x =>
{
x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
return(x);
})
.ToList();
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
historyBySymbolDailyOrHour.Add(symbol, history);
}
else
{
// group by date in DataTimeZone
var historyByDate = history.GroupBy(x => x.Time.Date).ToList();
historyBySymbol.Add(symbol, historyByDate);
}
}
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
SaveDailyOrHour(symbols, canonicalSymbol, historyBySymbolDailyOrHour);
}
else
{
SaveMinuteOrSecondOrTick(symbols, startTimeUtc, endTimeUtc, canonicalSymbol, historyBySymbol);
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, true);
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return(subscription);
}
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="algorithm">Existing algorithm instance</param>
/// <param name="brokerage">New brokerage instance</param>
/// <param name="baseJob">Backtesting job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configuration transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
//Set our default markets
algorithm.SetDefaultMarkets(BacktestingBrokerageFactory.DefaultMarketMap.ToDictionary());
algorithm.SetMaximumOrders(int.MaxValue);
// set our parameters
algorithm.SetParameters(baseJob.Parameters);
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Setup Base Algorithm:
algorithm.Initialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Add currency data feeds that weren't explicity added in Initialize
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, MarketHoursDatabase.FromDataFolder());
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
backtestJob.BacktestId = "LOCALHOST";
backtestJob.UserId = 1001;
backtestJob.Type = PacketType.BacktestNode;
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error(err);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
// set the transaction and settlement models based on the brokerage properties
SetupHandler.UpdateModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
return(initializeComplete);
}
/// <summary>
/// Returns an enumerator that defines when this user defined universe will be invoked
/// </summary>
/// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns>
public virtual IEnumerable <DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase)
{
var exchangeHours = marketHoursDatabase.GetExchangeHours(Configuration);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var first = true;
foreach (var dateTime in LinqExtensions.Range(localStartTime, localEndTime, dt => dt + Interval))
{
//Console.WriteLine("------------DATE TIME-------------------------------");
//Console.WriteLine(localStartTime + " " + localEndTime + " " + Interval);
//Console.WriteLine(dateTime);
if (first)
{
yield return(dateTime);
first = false;
}
else if (exchangeHours.IsOpen(dateTime, dateTime + Interval, Configuration.ExtendedMarketHours))
{
yield return(dateTime);
}
}
}
/// <summary>
/// Creates an instance of the PortfolioLooper class
/// </summary>
/// <param name="startingCash">Equity curve</param>
/// <param name="orders">Order events</param>
/// <param name="resolution">Optional parameter to override default resolution (Hourly)</param>
private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution)
{
// Initialize the providers that the HistoryProvider requires
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider");
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider");
_cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false);
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var dataPermissionManager = new DataPermissionManager();
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, _cacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false, dataPermissionManager));
Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders);
Algorithm.SetHistoryProvider(historyProvider);
// Dummy LEAN datafeed classes and initializations that essentially do nothing
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"");
var feed = new MockDataFeed();
// Create MHDB and Symbol properties DB instances for the DataManager
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
_dataManager = new DataManager(feed,
new UniverseSelection(
Algorithm,
new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
Algorithm,
Algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
_securityService = new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio));
var transactions = new BacktestingTransactionHandler();
_resultHandler = new BacktestingResultHandler();
// Initialize security services and other properties so that we
// don't get null reference exceptions during our re-calculation
Algorithm.Securities.SetSecurityService(_securityService);
Algorithm.SubscriptionManager.SetDataManager(_dataManager);
// Initializes all the proper Securities from the orders provided by the user
Algorithm.FromOrders(orders);
// Initialize the algorithm
Algorithm.Initialize();
Algorithm.PostInitialize();
// More initialization, this time with Algorithm and other misc. classes
_resultHandler.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions);
_resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
Algorithm.Transactions.SetOrderProcessor(transactions);
transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler);
feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null);
// Begin setting up the currency conversion feed if needed
var coreSecurities = Algorithm.Securities.Values.ToList();
if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto))
{
BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection);
var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList();
// Skip the history request if we don't need to convert anything
if (conversionSecurities.Any())
{
// Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency
_conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TimeTriggeredUniverseSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="universe">The user defined universe</param>
/// <param name="marketHoursDatabase">The market hours database</param>
/// <param name="timeProvider">The time provider</param>
public TimeTriggeredUniverseSubscriptionEnumeratorFactory(ITimeTriggeredUniverse universe, MarketHoursDatabase marketHoursDatabase, ITimeProvider timeProvider)
{
_universe = universe;
_timeProvider = timeProvider;
_marketHoursDatabase = marketHoursDatabase;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = config.Symbol
}).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
var cf = new CoarseFundamental();
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// load coarse data day by day
var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
let factory = new BaseDataSubscriptionFactory(config, date, false)
let source = cf.GetSource(config, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);
ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
// Will try to pull data from the data folder every 10min, file with yesterdays date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
TimeSpan.FromMinutes(10)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(config.Symbol, aggregator, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = Subscribe(subRequest.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = GetUniverseProvider(SecurityType.Option);
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = GetUniverseProvider(SecurityType.Future);
enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, _timeProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
enumerator = GetWarmupEnumerator(request, enumerator);
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
{
Subscribe(request.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
}
return(subscription);
}
public IQFeedFileHistoryProvider(LookupClient lookupClient, ISymbolMapper symbolMapper, MarketHoursDatabase marketHoursDatabase)
{
_lookupClient = lookupClient;
_symbolMapper = symbolMapper;
_marketHoursDatabase = marketHoursDatabase;
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager"></param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
Thread threadAlphas = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0)
{
IsBackground = true, Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
DataManager dataManager = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
_systemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
_algorithmHandlers.Alphas.Initialize(job, algorithm, _systemHandlers.Notify, _systemHandlers.Api);
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
(ISecurityInitializerProvider)algorithm);
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(_algorithmHandlers.DataFeed,
new UniverseSelection(
_algorithmHandlers.DataFeed,
algorithm,
securityService),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider,
_algorithmHandlers.FactorFileProvider, _algorithmHandlers.DataProvider, dataManager);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(_algorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
_algorithmHandlers.DataProvider,
historyDataCacheProvider,
_algorithmHandlers.MapFileProvider,
_algorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
}
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { _algorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { _algorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { _algorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
// alpha handler needs start/end dates to determine sample step sizes
_algorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Interpret(e.InnerException, _exceptionInterpreter);
message += _exceptionInterpreter.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + _algorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + _algorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + _algorithmHandlers.Alphas.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
_systemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run)
{
IsBackground = true, Name = "Transaction Thread"
};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run)
{
IsBackground = true, Name = "RealTime Thread"
};
threadAlphas = new Thread(() => _algorithmHandlers.Alphas.Run())
{
IsBackground = true, Name = "Alpha Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, dataManager, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _systemHandlers.LeanManager, _algorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
_systemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(_algorithmHandlers.Transactions, csvTransactionsFileName);
}
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) &&
charts.ContainsKey(benchmarkKey) &&
charts[benchmarkKey].Series.ContainsKey(benchmarkKey)
)
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ?
(algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue
: 0;
//Add other fixed parameters.
banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2"));
banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2"));
banner.Add("Return", netReturn.ToString("P"));
banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
}
}
catch (Exception err)
{
Log.Error(err, "Error generating statistics packet");
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
if (!_liveMode)
{
var kps = dataPoints / (double)1000 / totalSeconds;
_algorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
_algorithmHandlers.Alphas.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive ||
(_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) ||
(_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) ||
(_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive) ||
(_algorithmHandlers.Alphas != null && _algorithmHandlers.Alphas.IsActive)) &&
ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (threadAlphas != null && threadAlphas.IsAlive)
{
threadAlphas.Abort();
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (_algorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
public void TestBasicFeaturesWithOptionsFutures()
{
var securities = new SecurityManager(TimeKeeper);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
securities.Add(
Symbols.SPY,
new Security(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Equity, Symbols.SPY),
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
securities[Symbols.SPY].SetMarketPrice(new TradeBar {
Time = securities.UtcTime, Symbol = Symbols.SPY, Close = 195
});
var option1 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 192m, new DateTime(2016, 02, 16));
securities.Add(
option1,
new Option(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Option, option1),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var option2 = Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 193m, new DateTime(2016, 03, 19));
securities.Add(
option2,
new Option(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Option, option2),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var future1 = Symbol.CreateFuture(QuantConnect.Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2016, 02, 16));
securities.Add(
future1,
new Future(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Future, future1),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var future2 = Symbol.CreateFuture(QuantConnect.Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2016, 02, 19));
securities.Add(
future2,
new Future(
SecurityExchangeHours,
CreateTradeBarDataConfig(SecurityType.Future, future2),
new Cash(Currencies.USD, 0, 1m),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
)
);
var cal = new TradingCalendar(securities, marketHoursDatabase);
var optionDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count();
Assert.AreEqual(2, optionDays);
var futureDays = cal.GetDaysByType(TradingDayType.OptionExpiration, new DateTime(2016, 02, 16), new DateTime(2016, 03, 19)).Count();
Assert.AreEqual(2, futureDays);
var days = cal.GetTradingDays(new DateTime(2016, 02, 16), new DateTime(2016, 03, 19));
var optionAndfutureDays = days.Where(x => x.FutureExpirations.Any() || x.OptionExpirations.Any()).Count();
Assert.AreEqual(3, optionAndfutureDays);
// why? because option1 and future1 expire in one day 2016-02-16. Lets have a look.
var day = cal.GetTradingDay(new DateTime(2016, 02, 16));
Assert.AreEqual(1, day.OptionExpirations.Count());
Assert.AreEqual(1, day.FutureExpirations.Count());
var businessDays = days.Where(x => x.BusinessDay).Count();
Assert.AreEqual(24, businessDays);
var weekends = days.Where(x => x.Weekend).Count();
Assert.AreEqual(9, weekends);
Assert.AreEqual(24 + 9, (new DateTime(2016, 03, 19) - new DateTime(2016, 02, 16)).TotalDays + 1 /*inclusive*/);
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler,
IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider,
IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataProvider = dataProvider;
_dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider);
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public void ForexCashFills()
{
// this test asserts the portfolio behaves according to the Test_Cash algo, but for a Forex security,
// see TestData\CashTestingStrategy.csv; also "https://www.dropbox.com/s/oiliumoyqqj1ovl/2013-cash.csv?dl=1"
const string fillsFile = "TestData\\test_forex_fills.xml";
const string equityFile = "TestData\\test_forex_equity.xml";
const string mchQuantityFile = "TestData\\test_forex_fills_mch_quantity.xml";
const string jwbQuantityFile = "TestData\\test_forex_fills_jwb_quantity.xml";
var fills = XDocument.Load(fillsFile).Descendants("OrderEvent").Select(x => new OrderEvent(
x.Get <int>("OrderId"),
SymbolMap[x.Get <string>("Symbol")],
DateTime.MinValue,
x.Get <OrderStatus>("Status"),
x.Get <int>("FillQuantity") < 0 ? OrderDirection.Sell
: x.Get <int>("FillQuantity") > 0 ? OrderDirection.Buy
: OrderDirection.Hold,
x.Get <decimal>("FillPrice"),
x.Get <int>("FillQuantity"),
0)
).ToList();
var equity = XDocument.Load(equityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
var mchQuantity = XDocument.Load(mchQuantityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
var jwbQuantity = XDocument.Load(jwbQuantityFile).Descendants("decimal")
.Select(x => decimal.Parse(x.Value, CultureInfo.InvariantCulture))
.ToList();
Assert.AreEqual(fills.Count + 1, equity.Count);
// we're going to process fills and very our equity after each fill
var subscriptions = new SubscriptionManager(TimeKeeper);
var securities = new SecurityManager(TimeKeeper);
var transactions = new SecurityTransactionManager(securities);
var portfolio = new SecurityPortfolioManager(securities, transactions);
portfolio.SetCash(equity[0]);
portfolio.CashBook.Add("MCH", mchQuantity[0], 0);
portfolio.CashBook.Add("JWB", jwbQuantity[0], 0);
var jwbCash = portfolio.CashBook["JWB"];
var mchCash = portfolio.CashBook["MCH"];
var usdCash = portfolio.CashBook["USD"];
var mchJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, jwbCash, subscriptions.Add(MCHJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork));
mchJwbSecurity.SetLeverage(10m);
var mchUsdSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, usdCash, subscriptions.Add(MCHUSD, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork));
mchUsdSecurity.SetLeverage(10m);
var usdJwbSecurity = new QuantConnect.Securities.Forex.Forex(SecurityExchangeHours, mchCash, subscriptions.Add(USDJWB, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork));
usdJwbSecurity.SetLeverage(10m);
// no fee model
mchJwbSecurity.TransactionModel = new SecurityTransactionModel();
mchUsdSecurity.TransactionModel = new SecurityTransactionModel();
usdJwbSecurity.TransactionModel = new SecurityTransactionModel();
securities.Add(mchJwbSecurity);
securities.Add(usdJwbSecurity);
securities.Add(mchUsdSecurity);
portfolio.CashBook.EnsureCurrencyDataFeeds(securities, subscriptions, MarketHoursDatabase.FromDataFolder());
for (int i = 0; i < fills.Count; i++)
{
// before processing the fill we must deduct the cost
var fill = fills[i];
var time = DateTime.Today.AddDays(i);
// the value of 'MCJWB' increments for each fill, the original test algo did this monthly
// the time doesn't really matter though
decimal mchJwb = i + 1;
decimal mchUsd = (i + 1) / (i + 2m);
decimal usdJwb = i + 2;
Assert.AreEqual((double)mchJwb, (double)(mchUsd * usdJwb), 1e-10);
//Console.WriteLine("Step: " + i + " -- MCHJWB: " + mchJwb);
jwbCash.Update(new IndicatorDataPoint(MCHJWB, time, mchJwb));
usdCash.Update(new IndicatorDataPoint(MCHUSD, time, mchUsd));
mchCash.Update(new IndicatorDataPoint(JWBUSD, time, usdJwb));
var updateData = new Dictionary <Security, BaseData>
{
{ mchJwbSecurity, new IndicatorDataPoint(MCHJWB, time, mchJwb) },
{ mchUsdSecurity, new IndicatorDataPoint(MCHUSD, time, mchUsd) },
{ usdJwbSecurity, new IndicatorDataPoint(JWBUSD, time, usdJwb) }
};
foreach (var kvp in updateData)
{
kvp.Key.SetMarketPrice(kvp.Value);
}
portfolio.ProcessFill(fill);
//Console.WriteLine("-----------------------");
//Console.WriteLine(fill);
//Console.WriteLine("Post step: " + i);
//foreach (var cash in portfolio.CashBook)
//{
// Console.WriteLine(cash.Value);
//}
//Console.WriteLine("CashValue: " + portfolio.CashBook.TotalValueInAccountCurrency);
Console.WriteLine(i + 1 + " " + portfolio.TotalPortfolioValue.ToString("C"));
//Assert.AreEqual((double) equity[i + 1], (double)portfolio.TotalPortfolioValue, 2e-2);
Assert.AreEqual((double)mchQuantity[i + 1], (double)portfolio.CashBook["MCH"].Amount);
Assert.AreEqual((double)jwbQuantity[i + 1], (double)portfolio.CashBook["JWB"].Amount);
//Console.WriteLine();
//Console.WriteLine();
}
}
/// <summary>
/// Returns an enumerator that defines when this user defined universe will be invoked
/// </summary>
/// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns>
public IEnumerable <DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase)
{
var startTimeLocal = startTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone);
var endTimeLocal = endTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone);
// define date/time rule enumerable
var dates = _dateRule.GetDates(startTimeLocal, endTimeLocal);
foreach (var time in _timeRule.CreateUtcEventTimes(dates))
{
yield return(time);
}
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is ITimeTriggeredUniverse)
{
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_algorithm.UtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e),
_mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider));
}
if (request.Universe is FuturesChainUniverse)
{
return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
return(_subscriptionfactory);
}
public override void OnEndOfAlgorithm()
{
// Reset our Symbol property value, for testing purposes.
SymbolPropertiesDatabase.SetEntry(Market.USA, MarketHoursDatabase.GetDatabaseSymbolKey(_bitcoin.Symbol), SecurityType.Base,
SymbolProperties.GetDefault("USD"));
}
public void RefreshesOptionChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 19, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
TradeBarBuilderEnumerator underlyingEnumerator = null;
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
underlyingEnumerator = (TradeBarBuilderEnumerator)input;
return(new LiveFillForwardEnumerator(
timeProvider,
input,
option.Exchange,
fillForwardResolution,
false,
Time.EndOfTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
Time.BeginningOfTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new OptionChainUniverse(option, universeSettings, true);
var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, new DefaultDataProvider());
// 2018-10-19 10:00 AM UTC
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
// no underlying data available yet
Assert.IsNull(enumerator.Current);
Assert.AreEqual(0, symbolUniverse.TotalLookupCalls);
// 2018-10-19 10:01 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:02 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:03 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:03 AM UTC
timeProvider.Advance(Time.OneDay);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:04 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
enumerator.Dispose();
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
if (items == null || currentFrontierUtcTime == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
subscription.OnNewDataAvailable();
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm);
algorithm.Securities.SetSecurityService(securityService);
var dataManager = new DataManager(feed,
new UniverseSelection(algorithm, securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection, algorithm.Portfolio.CashBook);
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
synchronizer.SetSliceTimeZone(algorithm.TimeZone);
var totalDataPoints = 0;
var subscriptions = dataManager.DataFeedSubscriptions;
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// force JIT
synchronizer.Sync(subscriptions);
// log what we're doing
Console.WriteLine($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime = DateTime.MaxValue;
DateTime previousValue;
var stopwatch = Stopwatch.StartNew();
do
{
previousValue = currentTime;
var timeSlice = synchronizer.Sync(subscriptions);
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != previousValue);
stopwatch.Stop();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase,
bool liveMode)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
_liveMode = liveMode;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
universe.Dispose();
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is UserDefinedUniverse)
{
// Trigger universe selection when security added/removed after Initialize
var universe = (UserDefinedUniverse)request.Universe;
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Configuration.Type == typeof(FineFundamental))
{
return(new FineFundamentalSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return(new PostCreateConfigureSubscriptionEnumeratorFactory(
new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, _dataFileProvider, false, true),
enumerator => ConfigureEnumerator(request, false, enumerator)
));
}
/// <summary>
/// Adds the security to the user defined universe for the specified
/// </summary>
private void AddToUserDefinedUniverse(Security security)
{
// if we are adding a non-internal security which already has an internal feed, we remove it first
Security existingSecurity;
if (Securities.TryGetValue(security.Symbol, out existingSecurity))
{
if (!security.IsInternalFeed() && existingSecurity.IsInternalFeed())
{
var securityUniverse = UniverseManager.Select(x => x.Value).OfType <UserDefinedUniverse>().FirstOrDefault(x => x.Members.ContainsKey(security.Symbol));
securityUniverse?.Remove(security.Symbol);
Securities.Remove(security.Symbol);
}
}
Securities.Add(security);
// add this security to the user defined universe
Universe universe;
var subscription = security.Subscriptions.First();
var universeSymbol = UserDefinedUniverse.CreateSymbol(subscription.SecurityType, subscription.Market);
lock (_pendingUniverseAdditionsLock)
{
if (!UniverseManager.TryGetValue(universeSymbol, out universe))
{
universe = _pendingUniverseAdditions.FirstOrDefault(x => x.Configuration.Symbol == universeSymbol);
if (universe == null)
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var uconfig = new SubscriptionDataConfig(subscription, symbol: universeSymbol, isInternalFeed: true, fillForward: false);
if (security.Type == SecurityType.Base)
{
// set entry in market hours database for the universe subscription to match the custom data
var symbolString = MarketHoursDatabase.GetDatabaseSymbolKey(uconfig.Symbol);
MarketHoursDatabase.SetEntry(uconfig.Market, symbolString, uconfig.SecurityType, security.Exchange.Hours, uconfig.DataTimeZone);
}
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(security.Resolution, security.Leverage, security.IsFillDataForward, security.IsExtendedMarketHours,
TimeSpan.Zero),
SecurityInitializer,
QuantConnect.Time.MaxTimeSpan,
new List <Symbol> {
security.Symbol
}
);
_pendingUniverseAdditions.Add(universe);
}
}
}
var userDefinedUniverse = universe as UserDefinedUniverse;
if (userDefinedUniverse != null)
{
lock (_pendingUniverseAdditionsLock)
{
_pendingUserDefinedUniverseSecurityAdditions.Add(security, userDefinedUniverse);
}
}
else
{
// should never happen, someone would need to add a non-user defined universe with this symbol
throw new Exception("Expected universe with symbol '" + universeSymbol.Value + "' to be of type UserDefinedUniverse.");
}
}
public void AppliesDividendsOnce()
{
// init algorithm
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetLiveMode(true);
var dividend = new Dividend(Symbols.SPY, DateTime.UtcNow, 10m, 100m);
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
true,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
var synchronizer = new NullSynchronizer(algorithm, dividend);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(equities: new List <string> {
"SPY"
});
algorithm.Securities[Symbols.SPY].Holdings.SetHoldings(100m, 1);
algorithm.PostInitialize();
var initializedCash = algorithm.Portfolio.CashBook[Currencies.USD].Amount;
// init algorithm manager
var manager = new AlgorithmManager(true);
var job = new LiveNodePacket
{
UserId = 1,
ProjectId = 2,
DeployId = $"{nameof(PaperBrokerageTests)}.{nameof(AppliesDividendsOnce)}"
};
var results = new LiveTradingResultHandler();
var transactions = new BacktestingTransactionHandler();
var brokerage = new PaperBrokerage(algorithm, job);
// initialize results and transactions
results.Initialize(job, new EventMessagingHandler(), new Api.Api(), transactions);
results.SetAlgorithm(algorithm, algorithm.Portfolio.TotalPortfolioValue);
transactions.Initialize(algorithm, brokerage, results);
var realTime = new BacktestingRealTimeHandler();
// run algorithm manager
manager.Run(job,
algorithm,
synchronizer,
transactions,
results,
realTime,
new AlgorithmManagerTests.NullLeanManager(),
new AlgorithmManagerTests.NullAlphaHandler(),
new CancellationToken()
);
var postDividendCash = algorithm.Portfolio.CashBook[Currencies.USD].Amount;
realTime.Exit();
results.Exit();
Assert.AreEqual(initializedCash + dividend.Distribution, postDividendCash);
}