public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, dataManager, false, algorithm.Portfolio.CashBook);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer);
algorithm.Initialize();
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
feed.Exit();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
/// <summary>
/// Sets the initial cash for the algorithm if set in the job packet.
/// </summary>
/// <remarks>Should be called after initialize <see cref="LoadBacktestJobAccountCurrency"/></remarks>
public static void LoadBacktestJobCashAmount(IAlgorithm algorithm, BacktestNodePacket job)
{
// set initial cash, if present in the job
if (job.CashAmount.HasValue)
{
// Zero the CashBook - we'll populate directly from job
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
algorithm.SetCash(job.CashAmount.Value.Amount);
}
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_messagingHandler = messagingHandler;
_transactionHandler = transactionHandler;
_setupHandler = setupHandler;
_job = (BacktestNodePacket)job;
if (_job == null)
{
throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
}
_compileId = _job.CompileId;
_backtestId = _job.BacktestId;
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="transactionHandler">The transaction handler used to get the algorithms <see cref="Order"/> information</param>
public virtual void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, ITransactionHandler transactionHandler)
{
_algorithmId = job.AlgorithmId;
_projectId = job.ProjectId;
MessagingHandler = messagingHandler;
TransactionHandler = transactionHandler;
_job = (BacktestNodePacket)job;
if (_job == null)
{
throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
}
JobId = _job.BacktestId;
CompileId = _job.CompileId;
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
DataEndpoint = DataFeedEndpoint.LiveTrading,
RealTimeEndpoint = RealTimeEndpoint.LiveTrading,
ResultEndpoint = ResultHandlerEndpoint.Console,
SetupEndpoint = SetupHandlerEndpoint.Brokerage,
TransactionEndpoint = TransactionHandlerEndpoint.Brokerage,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
Channel = Config.Get("job-channel"),
UserId = Config.GetInt("job-user-id")
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(string.Format("JobQueue.NextJob(): Error resoliving BrokerageData for live job for brokerage {0}. {1}", liveJob.Brokerage, err.Message));
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
DataEndpoint = DataFeedEndpoint.FileSystem,
SetupEndpoint = SetupHandlerEndpoint.Console,
ResultEndpoint = ResultHandlerEndpoint.Console,
RealTimeEndpoint = RealTimeEndpoint.Backtesting,
TransactionEndpoint = TransactionHandlerEndpoint.Backtesting,
Algorithm = File.ReadAllBytes(AlgorithmLocation)
};
return(backtestJob);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider);
algorithm.Initialize();
algorithm.PostInitialize();
var feedThreadStarted = new ManualResetEvent(false);
var dataFeedThread = new Thread(() =>
{
feedThreadStarted.Set();
feed.Run();
})
{
IsBackground = true
};
dataFeedThread.Start();
feedThreadStarted.WaitOne();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
Log.Trace("JobQueue.NextJob(): Selected " + location);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
Channel = Config.Get("job-channel"),
UserId = Config.GetInt("job-user-id"),
Version = Constants.Version,
DeployId = Config.Get("algorithm-type-name"),
RamAllocation = int.MaxValue
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(string.Format("JobQueue.NextJob(): Error resoliving BrokerageData for live job for brokerage {0}. {1}", liveJob.Brokerage, err.Message));
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Version = Constants.Version,
BacktestId = Config.Get("algorithm-type-name"),
RamAllocation = int.MaxValue,
Language = (Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language"))
};
return(backtestJob);
}
public void TestAlgorithmManagerSpeed()
{
var algorithmManager = new AlgorithmManager(false);
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"{nameof(AlgorithmManagerTests)}.{nameof(TestAlgorithmManagerSpeed)}");
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null)),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var transactions = new BacktestingTransactionHandler();
var results = new BacktestingResultHandler();
var realtime = new BacktestingRealTimeHandler();
var leanManager = new NullLeanManager();
var alphas = new NullAlphaHandler();
var token = new CancellationToken();
var nullSynchronizer = new NullSynchronizer(algorithm);
algorithm.Initialize();
algorithm.PostInitialize();
results.Initialize(job, new QuantConnect.Messaging.Messaging(), new Api.Api(), transactions);
results.SetAlgorithm(algorithm, algorithm.Portfolio.TotalPortfolioValue);
transactions.Initialize(algorithm, new BacktestingBrokerage(algorithm), results);
feed.Initialize(algorithm, job, results, null, null, null, dataManager, null);
Log.Trace("Starting algorithm manager loop to process " + nullSynchronizer.Count + " time slices");
var sw = Stopwatch.StartNew();
algorithmManager.Run(job, algorithm, nullSynchronizer, transactions, results, realtime, leanManager, alphas, token);
sw.Stop();
var thousands = nullSynchronizer.Count / 1000d;
var seconds = sw.Elapsed.TotalSeconds;
Log.Trace("COUNT: " + nullSynchronizer.Count + " KPS: " + thousands / seconds);
}
public void RoundTripNullJobDates()
{
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"{nameof(BacktestNodePacketTests)}.Pepe");
var serialized = JsonConvert.SerializeObject(job);
var job2 = JsonConvert.DeserializeObject <BacktestNodePacket>(serialized);
Assert.AreEqual(job.BacktestId, job2.BacktestId);
Assert.AreEqual(job.Name, job2.Name);
Assert.IsNull(job.PeriodFinish);
Assert.IsNull(job.PeriodStart);
Assert.AreEqual(job.PeriodFinish, job2.PeriodFinish);
Assert.AreEqual(job.PeriodStart, job2.PeriodStart);
Assert.AreEqual(job.ProjectId, job2.ProjectId);
Assert.AreEqual(job.SessionId, job2.SessionId);
Assert.AreEqual(job.Language, job2.Language);
}
/// <summary>
/// Facade for the parent's NextJob() function.
/// We're exposing this in order to change the BacktestId.
/// </summary>
/// <param name="location"></param>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
Log.Trace("JobQueue.NextJob(): Selected " + location);
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
// We currently offer no live job functionality
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] { }, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(location),
HistoryProvider = Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"),
Channel = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = AlgorithmTypeName + "-" + Guid.NewGuid(),
Language = Language.CSharp,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
public void MessageHandler_WillSend_NewBackTestJob_ToCorrectRoute()
{
var backtest = new BacktestNodePacket();
using (var pullSocket = new PullSocket(">tcp://localhost:" + _port))
{
_messageHandler.SetAuthentication(backtest);
var message = pullSocket.ReceiveMultipartMessage();
var payload = message[0].ConvertToString();
var packet = JsonConvert.DeserializeObject <Packet>(payload);
Assert.IsTrue(message.FrameCount == 1);
Assert.IsTrue(PacketType.BacktestNode == packet.Type);
Assert.IsTrue(payload == JsonConvert.SerializeObject(backtest));
}
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Create a new backtesting data feed.
/// </summary>
/// <param name="algorithm">Instance of the algorithm</param>
/// <param name="job">Algorithm work task</param>
public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
{
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
DataFeed = DataFeedEndpoint.FileSystem;
IsActive = true;
Bridge = new ConcurrentQueue <List <BaseData> > [_subscriptions];
EndOfBridge = new bool[_subscriptions];
SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
//Class Privates:
_job = job;
_algorithm = algorithm;
_endOfStreams = false;
_bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_messagingHandler = messagingHandler;
_transactionHandler = transactionHandler;
_job = (BacktestNodePacket)job;
if (_job == null)
{
throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
}
_compileId = _job.CompileId;
_backtestId = _job.BacktestId;
//Get the resample period:
var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every
_resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00"));
}
/********************************************************
* PUBLIC CONSTRUCTOR
*********************************************************/
/// <summary>
/// Console result handler constructor.
/// </summary>
/// <remarks>Setup the default sampling and notification periods based on the backtest length.</remarks>
public ConsoleResultHandler(BacktestNodePacket packet)
{
Log.Trace("Launching Console Result Handler: QuantConnect v2.0");
Messages = new ConcurrentQueue <Packet>();
Charts = new ConcurrentDictionary <string, Chart>();
_chartLock = new Object();
_isActive = true;
_job = packet;
//Get the resample period:
const double samples = 4000;
const double minimumSamplePeriod = 4 * 60;
var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
var resampleMinutes = (totalMinutes < (minimumSamplePeriod * samples)) ? minimumSamplePeriod : (totalMinutes / samples); // Space out the sampling every
_resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
//Notification Period for pushes:
_notificationPeriod = TimeSpan.FromSeconds(5);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = new BenchmarkTest();
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider);
algorithm.Initialize();
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
var stopwatch = Stopwatch.StartNew();
var lastMonth = -1;
var count = 0;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper);
algorithm.SubscriptionManager.SetDataManager(dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
feed.Exit();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = GetAlgorithmLocation();
Log.Trace($"JobQueue.NextJob(): Selected {location}");
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python)
{
// Set the python path for loading python algorithms.
var pythonFile = new FileInfo(location);
var pythonPath = new List <string>
{
pythonFile.Directory.FullName,
new DirectoryInfo(Environment.CurrentDirectory).FullName,
};
// Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH
// environment variable won't be used in the module import process
var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH");
if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable))
{
pythonPath.Add(pythonPathEnvironmentVariable);
}
Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath));
}
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler),
DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
DeployId = AlgorithmTypeName,
Parameters = parameters,
Language = Language,
Controls = controls
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}");
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = AlgorithmTypeName,
Language = Language,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = GetAlgorithmLocation();
Log.Trace($"JobQueue.NextJob(): Selected {location}");
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python)
{
// Set the python path for loading python algorithms ("algorithm-location" config parameter)
var pythonFile = new FileInfo(location);
// PythonInitializer automatically adds the current working directory for us
PythonInitializer.SetPythonPathEnvironmentVariable(new string[] { pythonFile.Directory.FullName });
}
var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler),
DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
DeployId = algorithmId,
Parameters = parameters,
Language = Language,
Controls = controls
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}");
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = algorithmId,
Language = Language,
Parameters = parameters,
Controls = controls
};
string periodStart = Config.Get("period-start");
string periodFinish = Config.Get("period-finish");
string cashAmount = Config.Get("cash-amount");
if (!string.IsNullOrEmpty(periodStart))
{
backtestJob.PeriodStart = DateTime.Parse(periodStart, CultureInfo.InvariantCulture);
}
if (!string.IsNullOrEmpty(periodFinish))
{
backtestJob.PeriodFinish = DateTime.Parse(periodFinish, CultureInfo.InvariantCulture);
}
if (!string.IsNullOrEmpty(cashAmount))
{
decimal amount = Decimal.Parse(cashAmount, CultureInfo.InvariantCulture);
if (amount > 0)
{
backtestJob.CashAmount = new CashAmount(amount, Currencies.USD);
}
}
return(backtestJob);
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = GetAlgorithmLocation();
Log.Trace($"JobQueue.NextJob(): Selected {location}");
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler),
DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
OrganizationId = OrganizationId,
Version = Globals.Version,
DeployId = algorithmId,
Parameters = parameters,
Language = Language,
Controls = controls
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}");
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
OrganizationId = OrganizationId,
Version = Globals.Version,
BacktestId = algorithmId,
Language = Language,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
/// <summary>
/// Creates an instance of the PortfolioLooper class
/// </summary>
/// <param name="startingCash">Equity curve</param>
/// <param name="orders">Order events</param>
/// <param name="resolution">Optional parameter to override default resolution (Hourly)</param>
private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution)
{
// Initialize the providers that the HistoryProvider requires
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider");
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider");
_cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false);
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var dataPermissionManager = new DataPermissionManager();
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, _cacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false, dataPermissionManager));
Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders);
Algorithm.SetHistoryProvider(historyProvider);
// Dummy LEAN datafeed classes and initializations that essentially do nothing
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"");
var feed = new MockDataFeed();
// Create MHDB and Symbol properties DB instances for the DataManager
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
_dataManager = new DataManager(feed,
new UniverseSelection(
Algorithm,
new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
Algorithm,
Algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
_securityService = new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio));
var transactions = new BacktestingTransactionHandler();
_resultHandler = new BacktestingResultHandler();
// Initialize security services and other properties so that we
// don't get null reference exceptions during our re-calculation
Algorithm.Securities.SetSecurityService(_securityService);
Algorithm.SubscriptionManager.SetDataManager(_dataManager);
// Initializes all the proper Securities from the orders provided by the user
Algorithm.FromOrders(orders);
// Initialize the algorithm
Algorithm.Initialize();
Algorithm.PostInitialize();
// More initialization, this time with Algorithm and other misc. classes
_resultHandler.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions);
_resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
Algorithm.Transactions.SetOrderProcessor(transactions);
transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler);
feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null);
// Begin setting up the currency conversion feed if needed
var coreSecurities = Algorithm.Securities.Values.ToList();
if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto))
{
BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection);
var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList();
// Skip the history request if we don't need to convert anything
if (conversionSecurities.Any())
{
// Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency
_conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution);
}
}
}
/********************************************************
* CLASS PROPERTIES
*********************************************************/
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Create and connect to IQFeed.
/// </summary>
public IQFeedDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
: base(algorithm, job)
{
_algorithm = algorithm;
_job = job;
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
Log.Trace("JobQueue.NextJob(): Selected " + location);
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000)
};
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
Channel = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
DeployId = AlgorithmTypeName,
RamAllocation = int.MaxValue,
Parameters = parameters,
Language = Language,
Controls = controls
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, string.Format("Error resolving BrokerageData for live job for brokerage {0}:", liveJob.Brokerage));
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Channel = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = AlgorithmTypeName,
RamAllocation = int.MaxValue,
Language = Language,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
/********************************************************
* CLASS VARIABLES
*********************************************************/
/********************************************************
* CLASS PROPERTIES
*********************************************************/
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Pass through the backtesting datafeed to the underlying file system datafeed implementation.
/// </summary>
/// <param name="algorithm">Algorithm we're operating with</param>
/// <param name="job">Algorithm worker job</param>
public BacktestingDataFeed(IAlgorithm algorithm, BacktestNodePacket job) : base(algorithm, job)
{
DataFeed = DataFeedEndpoint.Backtesting;
}
/// <summary>
/// Resolve max orders for this algorithm
/// </summary>
/// <param name="job"></param>
protected override int GetMaximumOrders(BacktestNodePacket job)
{
// For local backtest MaxOrders is always max int
return(int.MaxValue);
}
public void FutureChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2013, 10, 07));
algorithm.SetEndDate(new DateTime(2013, 10, 08));
algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
future.SetFilter(0, 300);
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0L;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
{
// skip closed market times
currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
}
var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
var chainData = universeData[0].Value;
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
if (chainData.FilteredContracts.IsNullOrEmpty())
{
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
}
if (universeData.Count == 1)
{
// the chain
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
}
else
{
// we have 2 universe data, the chain and the continuous future
Assert.AreEqual(2, universeData.Count);
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
var continuousData = universeData[1].Value;
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
}
count++;
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 2 days worth of minute data
Assert.AreEqual(24 * 2 * 60 + 1, count);
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = GetAlgorithmLocation();
Log.Trace($"JobQueue.NextJob(): Selected {location}");
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var dataHandlers = Config.Get("data-queue-handler", DefaultDataQueueHandler);
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
DataQueueHandler = dataHandlers,
DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
OrganizationId = OrganizationId,
Version = Globals.Version,
DeployId = algorithmId,
Parameters = parameters,
Language = Language,
Controls = controls
};
Type brokerageName = null;
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
brokerageName = brokerageFactory.BrokerageType;
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}");
}
foreach (var dataHandlerName in dataHandlers.DeserializeList())
{
var brokerageFactoryForDataHandler = GetFactoryFromDataQueueHandler(dataHandlerName);
if (brokerageFactoryForDataHandler == null)
{
Log.Trace($"JobQueue.NextJob(): Not able to fetch data handler factory with name: {dataHandlerName}");
continue;
}
if (brokerageFactoryForDataHandler.BrokerageType == brokerageName)
{
//Don't need to add brokearageData again if added by brokerage
continue;
}
foreach (var data in brokerageFactoryForDataHandler.BrokerageData)
{
if (data.Key == "live-holdings")
{
//live-holdings not required for data handler
continue;
}
else if (!liveJob.BrokerageData.ContainsKey(data.Key))
{
liveJob.BrokerageData.Add(data.Key, data.Value);
}
else
{
throw new ArgumentException($"JobQueue.NextJob(): Key already exists in BrokerageData -- {data.Key}");
}
}
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] { }, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
OrganizationId = OrganizationId,
Version = Globals.Version,
BacktestId = algorithmId,
Language = Language,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
public BacktestConsoleStatusHandler(BacktestNodePacket _job)
{
this._job = _job;
}
