public void DataIsCachedCorrectly()
{
var reader = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
_config,
_initialDate,
false);
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
var dataBars = reader.Read(source).ToList();
var dataBars2 = reader.Read(source).ToList();
Assert.AreEqual(dataBars2.Count, dataBars.Count);
Assert.IsTrue(dataBars.SequenceEqual(dataBars2, new CustomComparer <BaseData>(
(data, baseData) =>
{
if (data.EndTime == baseData.EndTime &&
data.Time == baseData.Time &&
data.Symbol == baseData.Symbol &&
data.Price == baseData.Price &&
data.DataType == baseData.DataType &&
data.Value == baseData.Value)
{
return(0);
}
return(1);
})));
}
public void DataIsNotCachedForEphemeralDataCacheProvider()
{
var config = new SubscriptionDataConfig(
typeof(TestTradeBarFactory),
Symbol.Create("SymbolNonEphemeralTest1", SecurityType.Equity, Market.USA),
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var dataCacheProvider = new CustomEphemeralDataCacheProvider {
IsDataEphemeral = true
};
var reader = new TextSubscriptionDataSourceReader(
dataCacheProvider,
config,
_initialDate,
false);
var source = (new TradeBar()).GetSource(config, _initialDate, false);
dataCacheProvider.Data = "20000101 00:00,1,1,1,1,1";
var dataBars = reader.Read(source).First();
dataCacheProvider.Data = "20000101 00:00,2,2,2,2,2";
var dataBars2 = reader.Read(source).First();
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars.Time);
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars2.Time);
Assert.AreNotEqual(dataBars.Price, dataBars2.Price);
}
/// <summary>
/// Get fundamental data from given symbols
/// </summary>
/// <param name="pyObject">The symbols to retrieve fundamental data for</param>
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
/// <param name="start">The start date of selected data</param>
/// <param name="end">The end date of selected data</param>
/// <returns></returns>
public PyObject GetFundamental(PyObject tickers, string selector, DateTime?start = null, DateTime?end = null)
{
if (string.IsNullOrWhiteSpace(selector))
{
return("Invalid selector. Cannot be None, empty or consist only of white-space characters".ToPython());
}
using (Py.GIL())
{
// If tickers are not a PyList, we create one
if (!PyList.IsListType(tickers))
{
var tmp = new PyList();
tmp.Append(tickers);
tickers = tmp;
}
var list = new List <Tuple <Symbol, DateTime, object> >();
foreach (var ticker in tickers)
{
var symbol = QuantConnect.Symbol.Create(ticker.ToString(), SecurityType.Equity, Market.USA);
var dir = new DirectoryInfo(Path.Combine(Globals.DataFolder, "equity", symbol.ID.Market, "fundamental", "fine", symbol.Value.ToLowerInvariant()));
if (!dir.Exists)
{
continue;
}
var config = new SubscriptionDataConfig(typeof(FineFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
foreach (var fileName in dir.EnumerateFiles())
{
var date = DateTime.ParseExact(fileName.Name.Substring(0, 8), DateFormat.EightCharacter, CultureInfo.InvariantCulture);
if (date < start || date > end)
{
continue;
}
var factory = new TextSubscriptionDataSourceReader(_dataCacheProvider, config, date, false);
var source = new SubscriptionDataSource(fileName.FullName, SubscriptionTransportMedium.LocalFile);
var value = factory.Read(source).Select(x => GetPropertyValue(x, selector)).First();
list.Add(Tuple.Create(symbol, date, value));
}
}
var data = new PyDict();
foreach (var item in list.GroupBy(x => x.Item1))
{
var index = item.Select(x => x.Item2);
data.SetItem(item.Key, _pandas.Series(item.Select(x => x.Item3).ToList(), index));
}
return(_pandas.DataFrame(data));
}
}
public void CachedDataIsReturnedAsClone()
{
var reader = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
_config,
_initialDate,
false);
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
var dataBars = reader.Read(source).First();
dataBars.Value = 0;
var dataBars2 = reader.Read(source).First();
Assert.AreNotEqual(dataBars.Price, dataBars2.Price);
}
public void DataIsCorrect()
{
var date = new DateTime(2017, 10, 22);
var config = new SubscriptionDataConfig(
typeof(QuoteBar),
Symbol.Create("OANDA/EURUSD", SecurityType.Forex, Market.Oanda),
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
//var dataCacheProvider = new CustomEphemeralDataCacheProvider { IsDataEphemeral = true };
var dataProvider = new QuantConnect.Lean.Engine.DataFeeds.OandaDataProvider();
//dataProvider.Initialize(TestConfiguration.Parameters["token"], TestConfiguration.Parameters["dataPath"]);
var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider);
var reader = new TextSubscriptionDataSourceReader(
dataCacheProvider,
config,
date,
false);
Config.Set("oanda-data-access-token", TestConfiguration.Parameters["token"]);
Config.Set("data-folder", TestConfiguration.Parameters["dataPath"]);
Globals.Reset();
var source = (new ForexOandaVolume()).GetSource(config, date, false);
var dataBars = reader.Read(source);
decimal[] prices = { 1.176455m, 1.17648m };
BaseData[] data = dataBars.ToArray();
Assert.AreEqual(data[0].Price, prices[0]);
Assert.AreEqual(data[1].Price, prices[1]);
}
public void CacheBehaviorDifferentResolutions(Resolution resolution, bool shouldBeCached)
{
_config = new SubscriptionDataConfig(
typeof(TestTradeBarFactory),
Symbols.SPY,
resolution,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var reader = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider(), isDataEphemeral: false),
_config,
new DateTime(2013, 10, 07),
false);
var source = (new TradeBar()).GetSource(_config, new DateTime(2013, 10, 07), false);
// first call should cache
reader.Read(source).First();
TestTradeBarFactory.ReaderWasCalled = false;
reader.Read(source).First();
Assert.AreEqual(!shouldBeCached, TestTradeBarFactory.ReaderWasCalled);
}
public void RespectsInitialDate()
{
var reader = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
_config,
_initialDate,
false);
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
var dataBars = reader.Read(source).First();
Assert.Less(dataBars.EndTime, _initialDate);
// 80 days after _initialDate
var initialDate2 = _initialDate.AddDays(80);
var reader2 = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
_config,
initialDate2,
false);
var source2 = (new TradeBar()).GetSource(_config, initialDate2, false);
var dataBars2 = reader2.Read(source2).First();
Assert.Less(dataBars2.EndTime, initialDate2);
// 80 days before _initialDate
var initialDate3 = _initialDate.AddDays(-80);
var reader3 = new TextSubscriptionDataSourceReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
_config,
initialDate3,
false);
var source3 = (new TradeBar()).GetSource(_config, initialDate3, false);
var dataBars3 = reader3.Read(source3).First();
Assert.Less(dataBars3.EndTime, initialDate3);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager">The algorithm manager instance</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
/// <param name="workerThread">The worker thread instance</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread)
{
var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
TextSubscriptionDataSourceReader.SetCacheSize((int)(job.RamAllocation * 0.4));
//Reset thread holders.
var initializeComplete = false;
//-> Initialize messaging system
SystemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
IBrokerage brokerage = null;
DataManager dataManager = null;
IDataCacheProvider historyDataCacheProvider = null;
var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
try
{
// we get the mhdb before creating the algorithm instance,
// since the algorithm constructor will use it
var marketHoursDatabase = marketHoursDatabaseTask.Result;
AlgorithmHandlers.Setup.WorkerThread = workerThread;
// Save algorithm to cache, load algorithm instance:
algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
SystemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
// initialize the object store
AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls);
// initialize the data permission manager
AlgorithmHandlers.DataPermissionsManager.Initialize(job);
// notify the user of any errors w/ object store persistence
AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}");
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
registeredTypesProvider,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(AlgorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService,
AlgorithmHandlers.DataPermissionsManager,
AlgorithmHandlers.DataProvider),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
_liveMode,
registeredTypesProvider,
AlgorithmHandlers.DataPermissionsManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(algorithm, dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
AlgorithmHandlers.DataFeed.Initialize(
algorithm,
job,
AlgorithmHandlers.Results,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
AlgorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer,
AlgorithmHandlers.DataPermissionsManager.DataChannelProvider);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
SystemHandlers.Api,
AlgorithmHandlers.DataProvider,
historyDataCacheProvider,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
Invariant($"Processing history {progress}%..."));
}
},
// disable parallel history requests for live trading
parallelHistoryRequestsEnabled: !_liveMode,
dataPermissionManager: AlgorithmHandlers.DataPermissionsManager
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore));
// set this again now that we've actually added securities
AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
// alpha handler needs start/end dates to determine sample step sizes
AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = string.Join(",", algorithm.ErrorMessages);
errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value);
message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
AlgorithmHandlers.Results.RuntimeError(errorMessage);
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName);
Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
SystemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.UtcNow;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results);
AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
// Result manager scanning message queue: (started earlier)
AlgorithmHandlers.Results.DebugMessage(
$"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}");
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation, workerThread: workerThread);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F"));
throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
algorithm.RunTimeError = err;
algorithm.SetStatus(AlgorithmStatus.RuntimeError);
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
SystemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName);
}
if (!_liveMode)
{
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.UtcNow - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
var kps = dataPoints / (double)1000 / totalSeconds;
AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
dataManager?.RemoveAllSubscriptions();
workerThread?.Dispose();
}
synchronizer.DisposeSafely();
// Close data feed, alphas. Could be running even if algorithm initialization failed
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Alphas.Exit();
//Close result handler:
AlgorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var millisecondInterval = 10;
var millisecondTotalWait = 0;
while ((AlgorithmHandlers.Results.IsActive ||
(AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) ||
(AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) ||
(AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) ||
(AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) &&
millisecondTotalWait < 30 * 1000)
{
Thread.Sleep(millisecondInterval);
if (millisecondTotalWait % (millisecondInterval * 10) == 0)
{
Log.Trace("Waiting for threads to exit...");
}
millisecondTotalWait += millisecondInterval;
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (AlgorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
AlgorithmHandlers.Setup.Dispose();
}
historyDataCacheProvider.DisposeSafely();
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
AlgorithmHandlers.Results.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
}
}
public void CacheHappyPerformance()
{
long counter = 0;
var datas = new List <IEnumerable <BaseData> >();
var factory = new TradeBar();
var cacheProvider = new CustomEphemeralDataCacheProvider();
// we load SPY hour zip into memory and use it as the source of different fake tickers
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
Resolution.Hour,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var fakeSource = factory.GetSource(config, new DateTime(2013, 10, 07), false);
cacheProvider.Data = string.Join(Environment.NewLine, QuantConnect.Compression.ReadLines(fakeSource.Source));
for (var i = 0; i < 500; i++)
{
var ticker = $"{i}";
var fakeConfig = new SubscriptionDataConfig(
typeof(TradeBar),
Symbol.Create(ticker, SecurityType.Equity, Market.USA),
Resolution.Hour,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var reader = new TextSubscriptionDataSourceReader(cacheProvider, fakeConfig, Time.EndOfTime, false);
var source = factory.GetSource(fakeConfig, Time.BeginningOfTime, false);
datas.Add(reader.Read(source));
}
var timer = new Stopwatch();
timer.Start();
Parallel.ForEach(datas, enumerable =>
{
// after the first call should use the cache
foreach (var data in enumerable)
{
Interlocked.Increment(ref counter);
}
foreach (var data in enumerable)
{
Interlocked.Increment(ref counter);
}
foreach (var data in enumerable)
{
Interlocked.Increment(ref counter);
}
});
timer.Stop();
Log.Trace($"Took {timer.ElapsedMilliseconds}ms. Data count {counter}");
}
public void Performance(Type streamReaderType, Type readLineReaderType, TickType tickType)
{
var streamReaderMilliSeconds = 0L;
var streamReaderCount = 0;
var getLineReaderMilliSeconds = 0L;
var getLineReaderCount = 0;
var stopWatch = new Stopwatch();
{
var config = new SubscriptionDataConfig(
streamReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = streamReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
streamReaderCount = reader.Read(source).Count();
streamReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
streamReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
streamReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
{
var config = new SubscriptionDataConfig(
readLineReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = readLineReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
getLineReaderCount = reader.Read(source).Count();
getLineReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
getLineReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
getLineReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
Console.WriteLine($"StreamReader: {streamReaderMilliSeconds}ms. Count {streamReaderCount}");
Console.WriteLine($"GetLine Reader: {getLineReaderMilliSeconds}ms. Count {getLineReaderCount}");
// its 50% faster but lets leave some room to avoid noise
Assert.IsTrue((streamReaderMilliSeconds * 1.85d) < getLineReaderMilliSeconds);
Assert.AreEqual(getLineReaderCount, streamReaderCount);
}
