private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null)
{
getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2)
{
Quantity = 1
}));
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler(); // new ResultHandlerStub();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider));
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return(feed);
}
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution)
{
var start = new DateTime(2019, 12, 9);
var end = new DateTime(2019, 12, 12);
var mapFileProvider = new LocalDiskMapFileProvider();
var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA));
var dataReader = new SubscriptionDataReader(
new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
dataResolution,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false),
start,
end,
mapFileResolver,
new LocalDiskFactorFileProvider(mapFileProvider),
LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)),
false,
new TestDataCacheProvider
{
Data = data
}
);
Assert.IsTrue(dataReader.MoveNext());
Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext());
}
public void ChecksMapFileFirstDate()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var dataProvider = new DefaultDataProvider();
var resultHandler = new TestResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.AddEquity("AAA", Resolution.Daily);
var securityConfig = security.Subscriptions.First();
// start date is before the first date in the map file
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
new DateTime(2016, 11, 1));
var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider);
// should initialize the data source reader
enumerator.MoveNext();
enumerator.Dispose();
factory.DisposeSafely();
resultHandler.Exit();
var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
Assert.IsTrue(message.Equals(
"The starting date for symbol AAA, 2001-11-30, has been adjusted to match map file first date 2002-05-22."));
}
public override void Initialize()
{
SetStartDate(2014, 3, 25); //Set Start Date
SetEndDate(2014, 4, 7); //Set End Date
SetCash(100000); //Set Strategy Cash
// Set our DataNormalizationMode to raw
UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
_googl = AddEquity(Ticker, Resolution.Daily).Symbol;
// Get our factor file for this regression
var dataProvider =
Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider",
"DefaultDataProvider"));
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(dataProvider);
var factorFileProvider = new LocalDiskFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider;
// Prime our expected values
_expectedRawPrices.MoveNext();
}
public void RetrievesFromDisk()
{
var provider = new LocalDiskMapFileProvider();
var mapFiles = provider.Get(QuantConnect.Market.USA);
Assert.IsNotEmpty(mapFiles);
}
public void DividendsDistribution(string exDividendDateStr, decimal expectedDistribution)
{
var dividendProvider = new DividendEventProvider();
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
false, false, false);
var factorFile = new LocalDiskFactorFileProvider();
var mapFile = new LocalDiskMapFileProvider();
var start = new DateTime(1998, 01, 02);
dividendProvider.Initialize(config, factorFile.Get(Symbols.AAPL), mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start);
var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
var events = dividendProvider
.GetEvents(new NewTradableDateEventArgs(exDividendDate, null, Symbols.AAPL, null))
.ToList();
// ex dividend date does not emit anything
Assert.AreEqual(0, events.Count);
events = dividendProvider
.GetEvents(new NewTradableDateEventArgs(exDividendDate.AddDays(1), null, Symbols.AAPL, null))
.ToList();
Assert.AreEqual(1, events.Count);
var dividend = events[0] as Dividend;
Assert.IsNotNull(dividend);
Assert.AreEqual(expectedDistribution, dividend.Distribution);
}
private static IDataFeed RunLiveDataFeed(
IAlgorithm algorithm,
DateTime startDate,
IEnumerable <Symbol> symbols,
ITimeProvider timeProvider,
DataManager dataManager,
FuncDataQueueHandler funcDataQueueHandler = null)
{
var feed = new TestableLiveTradingDataFeed(funcDataQueueHandler ?? new FuncDataQueueHandler(x => Enumerable.Empty <BaseData>()),
timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), dataManager);
foreach (var symbol in symbols)
{
var config = algorithm.Securities[symbol].SubscriptionDataConfig;
var request = new SubscriptionRequest(false, null, algorithm.Securities[symbol], config, startDate, Time.EndOfTime);
feed.AddSubscription(request);
}
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
return(feed);
}
public void RetrievesFromDisk()
{
var provider = new LocalDiskMapFileProvider();
var mapFiles = provider.Get(VigiothCapital.QuantTrader.Market.USA);
Assert.IsNotEmpty(mapFiles);
}
public void CachesValueAndReturnsSameReference()
{
var provider = new LocalDiskMapFileProvider();
var factorFile1 = provider.Get(Symbols.SPY);
var factorFile2 = provider.Get(Symbols.SPY);
Assert.IsTrue(ReferenceEquals(factorFile1, factorFile2));
}
public void CachesValueAndReturnsSameReference()
{
var provider = new LocalDiskMapFileProvider();
var mapFiles1 = provider.Get(VigiothCapital.QuantTrader.Market.USA);
var mapFiles2 = provider.Get(VigiothCapital.QuantTrader.Market.USA);
Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2));
}
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null,
Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null, List <string> crypto = null)
{
_algorithm.SetStartDate(_startDate);
var lastTime = _manualTimeProvider.GetUtcNow();
getNextTicksFunction = getNextTicksFunction ?? (fdqh =>
{
var time = _manualTimeProvider.GetUtcNow();
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
var tickTime = lastTime.AddMinutes(-1).ConvertFromUtc(TimeZones.NewYork);
return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe
.Select(symbol => new Tick(tickTime, symbol, 1, 2)
{
Quantity = 1,
// Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here.
TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade
}).ToList());
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, _algorithm);
_algorithm.Securities.SetSecurityService(securityService);
_dataManager = new DataManager(feed,
new UniverseSelection(_algorithm, securityService),
_algorithm,
_algorithm.TimeKeeper,
marketHoursDatabase,
true);
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.AddSecurities(resolution, equities, forex, crypto);
_synchronizer = new TestableLiveSynchronizer(_manualTimeProvider);
_synchronizer.Initialize(_algorithm, _dataManager);
feed.Initialize(_algorithm, job, resultHandler, mapFileProvider,
new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, _dataManager, _synchronizer);
_algorithm.PostInitialize();
Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions
return(feed);
}
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null,
Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null)
{
_algorithm.SetStartDate(_startDate);
var lastTime = _manualTimeProvider.GetUtcNow();
getNextTicksFunction = getNextTicksFunction ?? (fdqh =>
{
var time = _manualTimeProvider.GetUtcNow();
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe
.Select(symbol => new Tick(lastTime.ConvertFromUtc(TimeZones.NewYork), symbol, 1, 2)
{
Quantity = 1,
// Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here.
TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade
}));
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, _manualTimeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
var dataManager = new DataManager(feed, new UniverseSelection(feed, _algorithm),
_algorithm.Settings, _algorithm.TimeKeeper);
_algorithm.SubscriptionManager.SetDataManager(dataManager);
_algorithm.AddSecurities(resolution, equities, forex);
feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager);
_algorithm.PostInitialize();
Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return(feed);
}
public void RetrievesFromDisk()
{
var provider = new LocalDiskMapFileProvider();
provider.Initialize(TestGlobals.DataProvider);
var mapFiles = provider.Get(AuxiliaryDataKey.EquityUsa);
Assert.IsNotEmpty(mapFiles);
}
public void InitializationSpeedTest()
{
var mapFileProvider = new LocalDiskMapFileProvider();
var sw = Stopwatch.StartNew();
var mapFileresolver = mapFileProvider.Get(Market.USA);
sw.Stop();
Console.WriteLine($"elapsed: {sw.Elapsed.TotalSeconds} seconds");
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider);
algorithm.Initialize();
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands / elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
feed.Exit();
dataManager.RemoveAllSubscriptions();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}
public void CachesValueAndReturnsSameReference()
{
var provider = new LocalDiskMapFileProvider();
provider.Initialize(TestGlobals.DataProvider);
var mapFiles1 = provider.Get(AuxiliaryDataKey.EquityUsa);
var mapFiles2 = provider.Get(AuxiliaryDataKey.EquityUsa);
Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2));
}
public void EmitsData()
{
var algorithm = new AlgorithmStub(forex: new List <string> {
"EURUSD"
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var dataProvider = new DefaultDataProvider();
var lastTime = DateTime.MinValue;
var timeProvider = new RealTimeProvider();
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
return(Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x * 100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m)));
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.Count != 0)
{
emittedData = true;
Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
Console.WriteLine();
}
});
Assert.IsTrue(emittedData);
}
public void EmitsBasedOnData()
{
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.AAPL,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var time = new DateTime(2021, 2, 17);
var timeProvider = new ManualTimeProvider(time);
Delisting delisting = null;
var autoResetEvent = new AutoResetEvent(false);
var dataQueueHandler = new FuncDataQueueHandler(handler =>
{
autoResetEvent.Set();
if (delisting != null)
{
var result = new[] { delisting };
delisting = null;
return(result);
}
return(Enumerable.Empty <BaseData>());
}, timeProvider);
var provider = new LiveDataBasedDelistingEventProvider(config, dataQueueHandler);
var mapFile = new LocalDiskMapFileProvider().Get(config.Symbol.ID.Market).ResolveMapFile(config.Symbol, config.Type);
provider.Initialize(config, null, mapFile, time);
Assert.IsTrue(autoResetEvent.WaitOne(TimeSpan.FromMilliseconds(100)));
var events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList();
Assert.AreEqual(0, events.Count);
delisting = new Delisting(Symbols.AAPL, time, 1, DelistingType.Warning);
Thread.Sleep(100);
events = provider.GetEvents(new NewTradableDateEventArgs(time.AddDays(-1), null, Symbols.AAPL, null)).ToList();
Assert.AreEqual(0, events.Count);
events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList();
Assert.AreEqual(1, events.Count);
Assert.AreEqual(time, (events[0] as Delisting).Time);
Assert.AreEqual(DelistingType.Warning, (events[0] as Delisting).Type);
events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList();
Assert.AreEqual(0, events.Count);
events = provider.GetEvents(new NewTradableDateEventArgs(time.AddDays(1), null, Symbols.AAPL, null)).ToList();
Assert.AreEqual(1, events.Count);
Assert.AreEqual(time.AddDays(1), (events[0] as Delisting).Time);
Assert.AreEqual(DelistingType.Delisted, (events[0] as Delisting).Type);
}
/// <summary>
/// Runs the Coarse universe generator with default values.
/// </summary>
/// <returns></returns>
public static bool CoarseUniverseGenerator()
{
var dailyDataFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, Resolution.Daily.ResolutionToLower()));
var destinationFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, "fundamental", "coarse"));
var blackListedTickersFile = new FileInfo("blacklisted-tickers.txt");
var reservedWordPrefix = Config.Get("reserved-words-prefix", "quantconnect-");
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var generator = new CoarseUniverseGeneratorProgram(dailyDataFolder, destinationFolder, Market.USA, blackListedTickersFile, reservedWordPrefix, mapFileProvider, factorFileProvider);
return generator.Run();
}
private void LaunchLean()
{
ConfigMerger.Merge(_config, _id, this.GetType());
var systemHandlers = new LeanEngineSystemHandlers(
new JobQueue(),
new EmptyApiHandler(),
new Messaging(),
new LocalLeanManager());
systemHandlers.Initialize();
//separate log uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt";
using (Log.LogHandler = new FileLogHandler(logFileName, true))
{
var map = new LocalDiskMapFileProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
new OptimizerResultHandler(),
new ConsoleSetupHandler(),
new FileSystemDataFeed(),
new BacktestingTransactionHandler(),
new BacktestingRealTimeHandler(),
map,
new LocalDiskFactorFileProvider(map),
new DefaultDataProvider(),
new OptimizerAlphaHandler(),
new EmptyObjectStore(),
new DataPermissionManager());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
try
{
var algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, algorithmPath, WorkerThread.Instance);
}
finally
{
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
public void FastExitsDoNotThrowUnhandledExceptions()
{
var algorithm = new AlgorithmStub();
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var feed = new TestableLiveTradingDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm);
algorithm.Securities.SetSecurityService(securityService);
var dataManager = new DataManager(feed,
new UniverseSelection(feed, algorithm, securityService),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick {
Symbol = SymbolCache.GetSymbol(x.ToString())
}).ToList();
feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager);
var unhandledExceptionWasThrown = false;
try
{
feed.Exit();
}
catch (Exception ex)
{
QuantConnect.Logging.Log.Error(ex.ToString());
unhandledExceptionWasThrown = true;
}
Thread.Sleep(500);
Assert.IsFalse(unhandledExceptionWasThrown);
}
public void ChecksFirstDate()
{
var mapFileProvider = new LocalDiskMapFileProvider();
var mapFileResolver = mapFileProvider.Get(Market.USA);
// QQQ started trading on 19990310
var mapFile = mapFileResolver.ResolveMapFile("QQQ", new DateTime(1999, 3, 9));
Assert.IsTrue(mapFile.IsNullOrEmpty());
var mapFile2 = mapFileResolver.ResolveMapFile("QQQ", new DateTime(2015, 3, 10));
Assert.IsFalse(mapFile2.IsNullOrEmpty());
}
public void FastExitsDoNotThrowUnhandledExceptions()
{
var algorithm = new AlgorithmStub();
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var feed = new TestableLiveTradingDataFeed();
var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm),
algorithm.Settings, algorithm.TimeKeeper);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick {
Symbol = SymbolCache.GetSymbol(x.ToString())
}).ToList();
feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager);
var feedThreadStarted = new ManualResetEvent(false);
var unhandledExceptionWasThrown = false;
Task.Run(() =>
{
try
{
feedThreadStarted.Set();
feed.Run();
}
catch (Exception ex)
{
QuantConnect.Logging.Log.Error(ex.ToString());
unhandledExceptionWasThrown = true;
}
});
feedThreadStarted.WaitOne();
feed.Exit();
Thread.Sleep(1000);
Assert.IsFalse(unhandledExceptionWasThrown);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper);
algorithm.SubscriptionManager.SetDataManager(dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
var feedThreadStarted = new ManualResetEvent(false);
var dataFeedThread = new Thread(() =>
{
feedThreadStarted.Set();
feed.Run();
})
{
IsBackground = true
};
dataFeedThread.Start();
feedThreadStarted.WaitOne();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
public void TestDataFeedEnumeratorStackSpeed()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var dataProvider = new DefaultDataProvider();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.Securities.Single().Value;
var securityConfig = security.Subscriptions.First();
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate);
var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider);
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
while (enumerator.MoveNext())
{
var current = enumerator.Current;
if (current == null)
{
Log.Trace("ERROR: Current is null");
continue;
}
if (current.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}");
lastMonth = current.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
enumerator.Dispose();
factory.DisposeSafely();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataManager = new DataManager(feed,
new UniverseSelection(feed,
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
feed.Exit();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
public void DoesNotLeakMemory()
{
var symbol = Symbols.AAPL;
var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market);
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider);
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(1998, 1, 1);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(false, null, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
enumerator.MoveNext();
}
date = date.AddDays(1);
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
public virtual void SetUp()
{
_algorithm = new QCAlgorithm();
_dataProvider = new DefaultDataProvider();
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(_dataProvider);
var factorFileProvider = new LocalZipFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, _dataProvider);
var historyProvider = new SubscriptionDataReaderHistoryProvider();
_cacheProvider = new ZipDataCacheProvider(_dataProvider);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null,
_dataProvider, _cacheProvider, mapFileProvider, factorFileProvider,
null, true, new DataPermissionManager()));
_algorithm.SetHistoryProvider(historyProvider);
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.Settings.FreePortfolioValue = 0;
}
public void ResolvesCorrectlyReUsedTicker()
{
var mapFileProvider = new LocalDiskMapFileProvider();
var mapFileResolver = mapFileProvider.Get(Market.USA);
// FB.1 started trading on 19990929 and ended on 20030328
var mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 28));
Assert.IsTrue(mapFile.IsNullOrEmpty());
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 29));
Assert.IsFalse(mapFile.IsNullOrEmpty());
// FB started trading on 20120518
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2012, 5, 17));
Assert.IsTrue(mapFile.IsNullOrEmpty());
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2015, 5, 18));
Assert.IsFalse(mapFile.IsNullOrEmpty());
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = new BenchmarkTest();
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider);
algorithm.Initialize();
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
var stopwatch = Stopwatch.StartNew();
var lastMonth = -1;
var count = 0;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
}
