/// <summary>
/// Indexer method for the security manager to access the securities objects by their symbol.
/// </summary>
/// <remarks>IDictionary implementation</remarks>
/// <param name="symbol">Symbol object indexer</param>
/// <returns>Security</returns>
public override Security this[Symbol symbol]
{
get
{
Security security;
if (!_securityManager.TryGetValue(symbol, out security))
{
throw new KeyNotFoundException($"This asset symbol ({symbol}) was not found in your security list. Please add this security or check it exists before using it with 'Securities.ContainsKey(\"{SymbolCache.GetTicker(symbol)}\")'");
}
return(security);
}
set
{
Security existing;
if (_securityManager.TryGetValue(symbol, out existing) && existing != value)
{
throw new ArgumentException($"Unable to over write existing Security: {symbol}");
}
// no security exists for the specified symbol key, add it now
if (existing == null)
{
Add(symbol, value);
}
}
}
static SymbolCache GetSymbolInfos (CancellationToken token)
{
getTypesTimer.BeginTiming ();
try {
var result = new SymbolCache ();
foreach (var workspace in TypeSystemService.AllWorkspaces) {
result.AddWorkspace (workspace, token);
}
return result;
} catch (AggregateException ae) {
ae.Flatten ().Handle (ex => ex is OperationCanceledException);
return SymbolCache.Empty;
} catch (OperationCanceledException) {
return SymbolCache.Empty;
} finally {
getTypesTimer.EndTiming ();
}
}
/// <summary>
/// Creates a security and matching configuration. This applies the default leverage if
/// leverage is less than or equal to zero.
/// This method also add the new symbol mapping to the <see cref="SymbolCache"/>
/// </summary>
public static Security CreateSecurity(List <Tuple <Type, TickType> > subscriptionDataTypes,
SecurityPortfolioManager securityPortfolioManager,
SubscriptionManager subscriptionManager,
SecurityExchangeHours exchangeHours,
DateTimeZone dataTimeZone,
SymbolProperties symbolProperties,
ISecurityInitializer securityInitializer,
Symbol symbol,
Resolution resolution,
bool fillDataForward,
decimal leverage,
bool extendedMarketHours,
bool isInternalFeed,
bool isCustomData,
bool isLiveMode,
bool addToSymbolCache = true,
bool isFilteredSubscription = true)
{
// add the symbol to our cache
if (addToSymbolCache)
{
SymbolCache.Set(symbol.Value, symbol);
}
// Add the symbol to Data Manager -- generate unified data streams for algorithm events
var configs = subscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward,
extendedMarketHours, isFilteredSubscription, isInternalFeed,
isCustomData, subscriptionDataTypes);
var configList = new SubscriptionDataConfigList(symbol);
configList.AddRange(configs);
// verify the cash book is in a ready state
var quoteCurrency = symbolProperties.QuoteCurrency;
if (!securityPortfolioManager.CashBook.ContainsKey(quoteCurrency))
{
// since we have none it's safe to say the conversion is zero
securityPortfolioManager.CashBook.Add(quoteCurrency, 0, 0);
}
if (symbol.ID.SecurityType == SecurityType.Forex || symbol.ID.SecurityType == SecurityType.Crypto)
{
// decompose the symbol into each currency pair
string baseCurrency;
Forex.Forex.DecomposeCurrencyPair(symbol.Value, out baseCurrency, out quoteCurrency);
if (!securityPortfolioManager.CashBook.ContainsKey(baseCurrency))
{
// since we have none it's safe to say the conversion is zero
securityPortfolioManager.CashBook.Add(baseCurrency, 0, 0);
}
if (!securityPortfolioManager.CashBook.ContainsKey(quoteCurrency))
{
// since we have none it's safe to say the conversion is zero
securityPortfolioManager.CashBook.Add(quoteCurrency, 0, 0);
}
}
var quoteCash = securityPortfolioManager.CashBook[symbolProperties.QuoteCurrency];
Security security;
switch (symbol.ID.SecurityType)
{
case SecurityType.Equity:
security = new Equity.Equity(symbol, exchangeHours, quoteCash, symbolProperties, securityPortfolioManager.CashBook);
break;
case SecurityType.Option:
if (addToSymbolCache)
{
SymbolCache.Set(symbol.Underlying.Value, symbol.Underlying);
}
security = new Option.Option(symbol, exchangeHours, securityPortfolioManager.CashBook[CashBook.AccountCurrency], new Option.OptionSymbolProperties(symbolProperties), securityPortfolioManager.CashBook);
break;
case SecurityType.Future:
security = new Future.Future(symbol, exchangeHours, securityPortfolioManager.CashBook[CashBook.AccountCurrency], symbolProperties, securityPortfolioManager.CashBook);
break;
case SecurityType.Forex:
security = new Forex.Forex(symbol, exchangeHours, quoteCash, symbolProperties, securityPortfolioManager.CashBook);
break;
case SecurityType.Cfd:
security = new Cfd.Cfd(symbol, exchangeHours, quoteCash, symbolProperties, securityPortfolioManager.CashBook);
break;
case SecurityType.Crypto:
security = new Crypto.Crypto(symbol, exchangeHours, quoteCash, symbolProperties, securityPortfolioManager.CashBook);
break;
default:
case SecurityType.Base:
security = new Security(symbol, exchangeHours, quoteCash, symbolProperties, securityPortfolioManager.CashBook);
break;
}
// if we're just creating this security and it only has an internal
// feed, mark it as non-tradable since the user didn't request this data
if (!isInternalFeed)
{
security.IsTradable = true;
}
security.AddData(configList);
// invoke the security initializer
securityInitializer.Initialize(security);
// if leverage was specified then apply to security after the initializer has run, parameters of this
// method take precedence over the intializer
if (leverage > 0)
{
security.SetLeverage(leverage);
}
// In live mode, equity assumes specific price variation model
if (isLiveMode && security.Type == SecurityType.Equity)
{
security.PriceVariationModel = new EquityPriceVariationModel();
}
return(security);
}
public void HandlesCoarseFundamentalData()
{
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
_algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero), SecurityInitializer.Null,
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
});
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
}
/// <summary>
/// Creates a new security
/// </summary>
/// <remarks>Following the obsoletion of Security.Subscriptions,
/// both overloads will be merged removing <see cref="SubscriptionDataConfig"/> arguments</remarks>
public Security CreateSecurity(Symbol symbol,
List <SubscriptionDataConfig> subscriptionDataConfigList,
decimal leverage = 0,
bool addToSymbolCache = true)
{
var configList = new SubscriptionDataConfigList(symbol);
configList.AddRange(subscriptionDataConfigList);
var exchangeHours = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours;
var defaultQuoteCurrency = CashBook.AccountCurrency;
if (symbol.ID.SecurityType == SecurityType.Forex || symbol.ID.SecurityType == SecurityType.Crypto)
{
defaultQuoteCurrency = symbol.Value.Substring(3);
}
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, defaultQuoteCurrency);
// add the symbol to our cache
if (addToSymbolCache)
{
SymbolCache.Set(symbol.Value, symbol);
}
// verify the cash book is in a ready state
var quoteCurrency = symbolProperties.QuoteCurrency;
if (!_cashBook.ContainsKey(quoteCurrency))
{
// since we have none it's safe to say the conversion is zero
_cashBook.Add(quoteCurrency, 0, 0);
}
if (symbol.ID.SecurityType == SecurityType.Forex || symbol.ID.SecurityType == SecurityType.Crypto)
{
// decompose the symbol into each currency pair
string baseCurrency;
Forex.Forex.DecomposeCurrencyPair(symbol.Value, out baseCurrency, out quoteCurrency);
if (!_cashBook.ContainsKey(baseCurrency))
{
// since we have none it's safe to say the conversion is zero
_cashBook.Add(baseCurrency, 0, 0);
}
if (!_cashBook.ContainsKey(quoteCurrency))
{
// since we have none it's safe to say the conversion is zero
_cashBook.Add(quoteCurrency, 0, 0);
}
}
var quoteCash = _cashBook[symbolProperties.QuoteCurrency];
Security security;
switch (symbol.ID.SecurityType)
{
case SecurityType.Equity:
security = new Equity.Equity(symbol, exchangeHours, quoteCash, symbolProperties, _cashBook);
break;
case SecurityType.Option:
if (addToSymbolCache)
{
SymbolCache.Set(symbol.Underlying.Value, symbol.Underlying);
}
security = new Option.Option(symbol, exchangeHours, _cashBook[CashBook.AccountCurrency], new Option.OptionSymbolProperties(symbolProperties), _cashBook);
break;
case SecurityType.Future:
security = new Future.Future(symbol, exchangeHours, _cashBook[CashBook.AccountCurrency], symbolProperties, _cashBook);
break;
case SecurityType.Forex:
security = new Forex.Forex(symbol, exchangeHours, quoteCash, symbolProperties, _cashBook);
break;
case SecurityType.Cfd:
security = new Cfd.Cfd(symbol, exchangeHours, quoteCash, symbolProperties, _cashBook);
break;
case SecurityType.Crypto:
security = new Crypto.Crypto(symbol, exchangeHours, quoteCash, symbolProperties, _cashBook);
break;
default:
case SecurityType.Base:
security = new Security(symbol, exchangeHours, quoteCash, symbolProperties, _cashBook);
break;
}
// if we're just creating this security and it only has an internal
// feed, mark it as non-tradable since the user didn't request this data
if (!configList.IsInternalFeed)
{
security.IsTradable = true;
}
security.AddData(configList);
// invoke the security initializer
_securityInitializerProvider.SecurityInitializer.Initialize(security);
// if leverage was specified then apply to security after the initializer has run, parameters of this
// method take precedence over the intializer
if (leverage > 0)
{
security.SetLeverage(leverage);
}
// In live mode, equity assumes specific price variation model
if (_isLiveMode && security.Type == SecurityType.Equity)
{
security.PriceVariationModel = new EquityPriceVariationModel();
}
return(security);
}
/// <summary>
/// Indexer method for the security manager to access the securities objects by their symbol.
/// </summary>
/// <remarks>IDictionary implementation</remarks>
/// <param name="symbol">Symbol object indexer</param>
/// <returns>Security</returns>
public Security this[Symbol symbol]
{
get
{
if (!_securityManager.ContainsKey(symbol))
{
throw new Exception(string.Format("This asset symbol ({0}) was not found in your security list. Please add this security or check it exists before using it with 'Securities.ContainsKey(\"{1}\")'", symbol, SymbolCache.GetTicker(symbol)));
}
return(_securityManager[symbol]);
}
set
{
Security existing;
if (_securityManager.TryGetValue(symbol, out existing) && existing != value)
{
throw new ArgumentException("Unable to over write existing Security: " + symbol.ToString());
}
// no security exists for the specified symbol key, add it now
if (existing == null)
{
Add(symbol, value);
}
}
}
public void AddDataSecurityTickerNoUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
{
SymbolCache.Clear();
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
Security asset;
switch (securityType)
{
case SecurityType.Cfd:
asset = qcAlgorithm.AddCfd(ticker, Resolution.Daily);
break;
case SecurityType.Crypto:
asset = qcAlgorithm.AddCrypto(ticker, Resolution.Daily);
break;
case SecurityType.Equity:
asset = qcAlgorithm.AddEquity(ticker, Resolution.Daily);
break;
case SecurityType.Forex:
asset = qcAlgorithm.AddForex(ticker, Resolution.Daily);
break;
case SecurityType.Future:
asset = qcAlgorithm.AddFuture(ticker, Resolution.Minute);
break;
default:
throw new Exception($"SecurityType {securityType} is not valid for this test");
}
// Dummy here is meant to try to corrupt the SymbolCache. Ideally, SymbolCache should return non-custom data types with higher priority
// in case we want to add two custom data types, but still have them associated with the equity from the cache if we're using it.
// This covers the case where two idential data subscriptions are created.
var dummy = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
var customData = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
// Check to see if we have an underlying symbol when we shouldn't
Assert.IsFalse(customData.Symbol.HasUnderlying, $"{customDataType.Name} has underlying symbol for SecurityType {securityType} with ticker {ticker}");
Assert.AreEqual(customData.Symbol.Underlying, null, $"{customDataType.Name} - Custom data underlying Symbol for SecurityType {securityType} is not null");
var assetSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First();
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Base).Single();
var assetShouldBeMapped = assetSubscription.TickerShouldBeMapped();
var customShouldBeMapped = customDataSubscription.TickerShouldBeMapped();
Assert.AreEqual(securityShouldBeMapped, assetShouldBeMapped);
Assert.AreEqual(customDataShouldBeMapped, customShouldBeMapped);
Assert.AreNotEqual(assetSubscription, customDataSubscription);
if (assetShouldBeMapped == customShouldBeMapped)
{
// Would fail with CL future without this check because MappedSymbol returns "/CL" for the Future symbol
if (assetSubscription.SecurityType == SecurityType.Future)
{
Assert.AreNotEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreNotEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
else
{
Assert.AreEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
}
}
public void AddDataSecurityTickerWithUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
{
SymbolCache.Clear();
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
Security asset;
switch (securityType)
{
case SecurityType.Cfd:
asset = qcAlgorithm.AddCfd(ticker, Resolution.Daily);
break;
case SecurityType.Crypto:
asset = qcAlgorithm.AddCrypto(ticker, Resolution.Daily);
break;
case SecurityType.Equity:
asset = qcAlgorithm.AddEquity(ticker, Resolution.Daily);
break;
case SecurityType.Forex:
asset = qcAlgorithm.AddForex(ticker, Resolution.Daily);
break;
case SecurityType.Future:
asset = qcAlgorithm.AddFuture(ticker, Resolution.Minute);
break;
default:
throw new Exception($"SecurityType {securityType} is not valid for this test");
}
// Aliased value for Futures contains a forward-slash, which causes the
// lookup in the SymbolCache to fail
if (securityType == SecurityType.Future)
{
ticker = asset.Symbol.Value;
}
// Dummy here is meant to try to corrupt the SymbolCache. Ideally, SymbolCache should return non-custom data types with higher priority
// in case we want to add two custom data types, but still have them associated with the equity from the cache if we're using it.
// This covers the case where two idential data subscriptions are created.
var dummy = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
var customData = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
Assert.IsTrue(customData.Symbol.HasUnderlying, $"Custom data added as {ticker} Symbol with SecurityType {securityType} does not have underlying");
Assert.AreEqual(customData.Symbol.Underlying, asset.Symbol, $"Custom data underlying does not match {securityType} Symbol for {ticker}");
var assetSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First();
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Base).Single();
var assetShouldBeMapped = assetSubscription.TickerShouldBeMapped();
var customShouldBeMapped = customDataSubscription.TickerShouldBeMapped();
if (securityType == SecurityType.Equity)
{
Assert.AreEqual(securityShouldBeMapped, assetShouldBeMapped);
Assert.AreEqual(customDataShouldBeMapped, customShouldBeMapped);
Assert.AreNotEqual(assetSubscription, customDataSubscription);
if (assetShouldBeMapped == customShouldBeMapped)
{
Assert.AreEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
}
}
public static AlgorithmManager RunLocalBacktest(
string algorithm,
Dictionary <string, string> expectedStatistics,
AlphaRuntimeStatistics expectedAlphaStatistics,
Language language,
AlgorithmStatus expectedFinalStatus,
DateTime?startDate = null,
DateTime?endDate = null,
string setupHandler = "RegressionSetupHandlerWrapper",
decimal?initialCash = null)
{
AlgorithmManager algorithmManager = null;
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics(new TestAccountCurrencyProvider());
Composer.Instance.Reset();
SymbolCache.Clear();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", setupHandler);
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.Trace("");
Log.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
try
{
string algorithmPath;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out algorithmPath);
job.BacktestId = algorithm;
job.PeriodStart = startDate;
job.PeriodFinish = endDate;
if (initialCash.HasValue)
{
job.CashAmount = new CashAmount(initialCash.Value, Currencies.USD);
}
algorithmManager = new AlgorithmManager(false, job);
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, algorithmPath);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath;
}
catch (Exception e)
{
Log.Trace($"Error in AlgorithmRunner task: {e}");
}
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
if (expectedFinalStatus != AlgorithmStatus.RuntimeError)
{
Log.Error("{0} {1}", ex.Message, ex.StackTrace);
}
}
if (algorithmManager?.State != expectedFinalStatus)
{
Assert.Fail($"Algorithm state should be {expectedFinalStatus} and is: {algorithmManager?.State}");
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
return(algorithmManager);
}
/// <summary>
/// Creates a security and matching configuration. This applies the default leverage if
/// leverage is less than or equal to zero.
/// This method also add the new symbol mapping to the <see cref="SymbolCache"/>
/// </summary>
public static Security CreateSecurity(Type factoryType,
SecurityPortfolioManager securityPortfolioManager,
SubscriptionManager subscriptionManager,
SecurityExchangeHours exchangeHours,
DateTimeZone dataTimeZone,
Symbol symbol,
Resolution resolution,
bool fillDataForward,
decimal leverage,
bool extendedMarketHours,
bool isInternalFeed,
bool isCustomData,
bool addToSymbolCache = true)
{
var sid = symbol.ID;
//If it hasn't been set, use some defaults based on the security type
if (leverage <= 0)
{
if (sid.SecurityType == SecurityType.Equity)
{
leverage = 2;
}
else if (sid.SecurityType == SecurityType.Forex)
{
leverage = 50;
}
// default to 1 for everything else
else
{
leverage = 1m;
}
}
// add the symbol to our cache
if (addToSymbolCache)
{
SymbolCache.Set(symbol.Value, symbol);
}
//Add the symbol to Data Manager -- generate unified data streams for algorithm events
var config = subscriptionManager.Add(factoryType, symbol, resolution, dataTimeZone, exchangeHours.TimeZone, isCustomData, fillDataForward,
extendedMarketHours, isInternalFeed);
Security security;
switch (config.SecurityType)
{
case SecurityType.Equity:
security = new Equity.Equity(exchangeHours, config, leverage);
break;
case SecurityType.Forex:
// decompose the symbol into each currency pair
string baseCurrency, quoteCurrency;
Forex.Forex.DecomposeCurrencyPair(symbol.Value, out baseCurrency, out quoteCurrency);
if (!securityPortfolioManager.CashBook.ContainsKey(baseCurrency))
{
// since we have none it's safe to say the conversion is zero
securityPortfolioManager.CashBook.Add(baseCurrency, 0, 0);
}
if (!securityPortfolioManager.CashBook.ContainsKey(quoteCurrency))
{
// since we have none it's safe to say the conversion is zero
securityPortfolioManager.CashBook.Add(quoteCurrency, 0, 0);
}
security = new Forex.Forex(exchangeHours, securityPortfolioManager.CashBook[quoteCurrency], config, leverage);
break;
default:
case SecurityType.Base:
security = new Security(exchangeHours, config, leverage);
break;
}
return(security);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataFileProvider = new DefaultDataFileProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
var request = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc);
var enumerator = factory.CreateEnumerator(request, dataFileProvider);
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc))
{
if (subscription.IsUniverseSubscription)
{
removals.Remove(member);
}
else
{
_dataFeed.RemoveSubscription(subscription.Configuration);
}
}
// remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD!
SymbolCache.TryRemove(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedMember = universe.AddMember(dateTimeUtc, security);
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = addedMember;
}
}
if (addedSubscription)
{
additions.Add(security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}