public void TickResolutionOpenInterestHistoryRequestIsFilteredByDefault_SingleSymbol()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var dataProvider = new DefaultDataProvider();
var zipCacheProvider = new ZipDataCacheProvider(dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
dataProvider,
zipCacheProvider,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
var start = new DateTime(2014, 6, 05);
_algorithm.SetStartDate(start);
_algorithm.SetDateTime(start.AddDays(2));
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var result = _algorithm.History(new[] { optionSymbol }, start, start.AddDays(2), Resolution.Minute, fillForward: false).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(result);
Assert.IsTrue(result.Any(slice => slice.ContainsKey(optionSymbol)));
var openInterests = result.Select(slice => slice.Get(typeof(OpenInterest)) as DataDictionary <OpenInterest>).Where(dataDictionary => dataDictionary.Count > 0).ToList();
Assert.AreEqual(0, openInterests.Count);
}
public void GetLastKnownPriceOfIlliquidAsset_RealData()
{
var algorithm = new QCAlgorithm();
algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm));
algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider());
algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
new DefaultDataProvider(),
cacheProvider,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
algorithm.SetDateTime(new DateTime(2014, 6, 6, 15, 0, 0));
//20140606_twx_minute_quote_american_call_230000_20150117.csv
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var option = algorithm.AddOptionContract(optionSymbol);
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.IsNotNull(lastKnownPrice);
// Data gap of more than 15 minutes
Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15);
cacheProvider.DisposeSafely();
}
public void TickResolutionOpenInterestHistoryRequestIsNotFilteredWhenRequestedExplicitly()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var zipCacheProvider = new ZipDataCacheProvider(_dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
_dataProvider,
zipCacheProvider,
_mapFileProvider,
_factorFileProvider,
null,
false,
new DataPermissionManager()));
var start = new DateTime(2014, 6, 05);
_algorithm.SetStartDate(start);
_algorithm.SetDateTime(start.AddDays(2));
_algorithm.UniverseSettings.FillForward = false;
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var openInterests = _algorithm.History <OpenInterest>(optionSymbol, start, start.AddDays(2), Resolution.Minute).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(openInterests);
Assert.AreEqual(2, openInterests.Count);
Assert.AreEqual(new DateTime(2014, 06, 05, 6, 32, 0), openInterests[0].Time);
Assert.AreEqual(optionSymbol, openInterests[0].Symbol);
Assert.AreEqual(new DateTime(2014, 06, 06, 6, 32, 0), openInterests[1].Time);
Assert.AreEqual(optionSymbol, openInterests[1].Symbol);
}
public void ChecksMapFileFirstDate()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var resultHandler = new TestResultHandler();
using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.AddEquity("AAA", Resolution.Daily);
var securityConfig = security.Subscriptions.First();
// start date is before the first date in the map file
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
new DateTime(2016, 11, 1));
var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
// should initialize the data source reader
enumerator.MoveNext();
enumerator.Dispose();
factory.DisposeSafely();
resultHandler.Exit();
var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
Assert.IsTrue(message.Equals(
"The starting dates for the following symbols have been adjusted to match their map files first date: [AAA, 2020-09-09]"));
}
public void TickResolutionHistoryRequest()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var zipCacheProvider = new ZipDataCacheProvider(_dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
_dataProvider,
zipCacheProvider,
_mapFileProvider,
_factorFileProvider,
null,
false,
new DataPermissionManager()));
_algorithm.SetStartDate(2013, 10, 08);
var start = new DateTime(2013, 10, 07);
// Trades and quotes
var result = _algorithm.History(new [] { Symbols.SPY }, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList();
// Just Trades
var result2 = _algorithm.History <Tick>(Symbols.SPY, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(result);
Assert.IsNotEmpty(result2);
Assert.IsTrue(result2.All(tick => tick.TickType == TickType.Trade));
// (Trades and quotes).Count > Trades * 2
Assert.Greater(result.Count, result2.Count * 2);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
// We decide to use the ZipDataCacheProvider instead of the SingleEntryDataCacheProvider here
// for resiliency and as a fix for an issue preventing us from reading non-equity options data.
// It has the added benefit of caching any zip files that we request from the filesystem, and reading
// files contained within the zip file, which the SingleEntryDataCacheProvider does not support.
var sourceFactory = request.Configuration.GetBaseDataInstance();
using (var dataCacheProvider = new ZipDataCacheProvider(dataProvider))
{
foreach (var date in _tradableDaysProvider(request))
{
if (sourceFactory.RequiresMapping())
{
request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
}
var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
}
public void ZipDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral)
{
var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), isDataEphemeral: isDataEphemeral);
var remoteReader = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
var remoteReader2 = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
remoteReader.Dispose();
remoteReader2.Dispose();
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceOption()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09));
var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15)));
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void OptionsAreMappedCorrectly()
{
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCache,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
var symbol = Symbol.CreateOption(
"FOXA",
Market.USA,
OptionStyle.American,
OptionRight.Call,
32,
new DateTime(2013, 07, 20));
var result = historyProvider.GetHistory(
new[]
{
new HistoryRequest(new DateTime(2013, 06, 28),
new DateTime(2013, 07, 03),
typeof(QuoteBar),
symbol,
Resolution.Minute,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
TickType.Quote)
},
TimeZones.NewYork).ToList();
Assert.IsNotEmpty(result);
// assert we fetch the data for the previous and new symbol
var firstBar = result.First().Values.Single();
var lastBar = result.Last().Values.Single();
Assert.IsTrue(firstBar.Symbol.Value.Contains("NWSA"));
Assert.AreEqual(28, firstBar.Time.Date.Day);
Assert.IsTrue(lastBar.Symbol.Value.Contains("FOXA"));
Assert.AreEqual(2, lastBar.Time.Date.Day);
zipCache.DisposeSafely();
}
public void GetLastKnownPriceEquity()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var equity = algorithm.AddEquity("SPY");
var lastKnownPrice = algorithm.GetLastKnownPrice(equity);
Assert.AreEqual(typeof(TradeBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceFuture()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var future = algorithm.AddSecurity(Symbols.CreateFutureSymbol(Futures.Indices.SP500EMini, new DateTime(2013, 12, 20)));
var lastKnownPrice = algorithm.GetLastKnownPrice(future);
Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceOfCustomData()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2018, 4, 4));
var alpha = algorithm.AddData <AlphaStreamsPortfolioState>("9fc8ef73792331b11dbd5429a");
var lastKnownPrice = algorithm.GetLastKnownPrice(alpha);
Assert.IsNotNull(lastKnownPrice);
cacheProvider.DisposeSafely();
}
public void MultiThreadReadWriteTest()
{
var dataCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider, cacheTimer: 0.1);
Parallel.For(0, 100, (i) =>
{
var data = new byte[300];
_random.NextBytes(data);
ReadAndWrite(dataCacheProvider, data);
});
dataCacheProvider.Dispose();
}
public virtual void SetUp()
{
_algorithm = new QCAlgorithm();
var historyProvider = new SubscriptionDataReaderHistoryProvider();
_cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null,
TestGlobals.DataProvider, _cacheProvider, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider,
null, true, new DataPermissionManager()));
_algorithm.SetHistoryProvider(historyProvider);
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.Settings.FreePortfolioValue = 0;
_algorithm.SetFinishedWarmingUp();
}
/// <summary>
/// Set up the HistoryProvider for algorithm
/// </summary>
protected void SetUpHistoryProvider()
{
Algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
ZipCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
Algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
ZipCacheProvider,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
}
public void GetLastKnownPricesEquity()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var equity = algorithm.AddEquity("SPY");
var lastKnownPrices = algorithm.GetLastKnownPrices(equity.Symbol).ToList();
Assert.AreEqual(2, lastKnownPrices.Count);
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar)));
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar)));
cacheProvider.DisposeSafely();
}
public void GetLastKnownPricesOption()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09));
var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15)));
var lastKnownPrices = algorithm.GetLastKnownPrices(option).ToList();;
Assert.AreEqual(2, lastKnownPrices.Count);
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar)));
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar)));
cacheProvider.DisposeSafely();
}
public void StoreFailsCorruptedFile()
{
var dataCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider, cacheTimer: 0.1);
var tempZipFileEntry = Path.GetTempFileName().Replace(".tmp", ".zip", StringComparison.InvariantCulture);
var data = new byte[300];
_random.NextBytes(data);
File.WriteAllText(tempZipFileEntry, "corrupted zip");
Assert.Throws <InvalidOperationException>(() => dataCacheProvider.Store(tempZipFileEntry + "#testEntry.csv", data));
dataCacheProvider.Dispose();
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="resultHandler">The result handler for the algorithm</param>
/// <param name="mapFileProvider">The map file provider</param>
/// <param name="factorFileProvider">The factor file provider</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param>
/// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
/// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
bool includeAuxiliaryData,
Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null
)
{
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
_isLiveMode = false;
_includeAuxiliaryData = includeAuxiliaryData;
_tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
}
private List <Symbol> LoadFutureChain(Symbol baseFuture, DateTime date, TickType tickType, Resolution res)
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, baseFuture, date, res, tickType);
//load future chain first
var config = new SubscriptionDataConfig(typeof(ZipEntryName), baseFuture, res,
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, date, false);
var result = factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName))
.Select(s => s.Symbol).ToList();
return(result);
}
public void TestDataFeedEnumeratorStackSpeed()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var resultHandler = new BacktestingResultHandler();
using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.Securities.Single().Value;
var securityConfig = security.Subscriptions.First();
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate);
var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
while (enumerator.MoveNext())
{
var current = enumerator.Current;
if (current == null)
{
Log.Trace("ERROR: Current is null");
continue;
}
if (current.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}");
lastMonth = current.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
enumerator.Dispose();
factory.DisposeSafely();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}
public void WarmUpPythonIndicatorProperly()
{
var algo = new AlgorithmStub
{
HistoryProvider = new SubscriptionDataReaderHistoryProvider()
};
var zipCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
algo.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCacheProvider,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
algo.SetStartDate(2013, 10, 08);
algo.AddEquity("SPY", Resolution.Minute);
// Different types of indicators
var indicatorDataPoint = new SimpleMovingAverage("SPY", 10);
var indicatorDataBar = new AverageTrueRange("SPY", 10);
var indicatorTradeBar = new VolumeWeightedAveragePriceIndicator("SPY", 10);
using (Py.GIL())
{
var sma = indicatorDataPoint.ToPython();
var atr = indicatorTradeBar.ToPython();
var vwapi = indicatorDataBar.ToPython();
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", sma, Resolution.Minute));
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", atr, Resolution.Minute));
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", vwapi, Resolution.Minute));
var smaIsReady = ((dynamic)sma).IsReady;
var atrIsReady = ((dynamic)atr).IsReady;
var vwapiIsReady = ((dynamic)vwapi).IsReady;
Assert.IsTrue(smaIsReady.IsTrue());
Assert.IsTrue(atrIsReady.IsTrue());
Assert.IsTrue(vwapiIsReady.IsTrue());
}
zipCacheProvider.DisposeSafely();
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="resultHandler">The result handler for the algorithm</param>
/// <param name="mapFileProvider">The map file provider</param>
/// <param name="factorFileProvider">The factor file provider</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
/// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
/// <param name="enablePriceScaling">Applies price factor</param>
public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null,
bool enablePriceScaling = true
)
{
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_numericalPrecisionMessageSent = new ConcurrentSet <Symbol>();
_zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
_isLiveMode = false;
_tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
_enablePriceScaling = enablePriceScaling;
}
private static ISubscriptionDataSourceReader Initialize(bool liveMode, Resolution resolution, out SubscriptionDataSource source)
{
using var dataProvider = new DefaultDataProvider();
using var cache = new ZipDataCacheProvider(dataProvider);
var config = new SubscriptionDataConfig(typeof(TestBaseDataCollection),
Symbols.SPY,
resolution,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false);
var date = DateTime.MinValue;
var path = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, resolution, TickType.Trade);
source = new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile);
return(new BaseDataCollectionAggregatorReader(cache, config, date, liveMode));
}
public void GetLastKnownPriceOfIlliquidAsset_RealData()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 6, 6, 11, 0, 0));
//20140606_twx_minute_quote_american_call_230000_20150117.csv
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var option = algorithm.AddOptionContract(optionSymbol);
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.IsNotNull(lastKnownPrice);
// Data gap of more than 15 minutes
Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15);
cacheProvider.DisposeSafely();
}
public void DoesNotLeakMemory()
{
var symbol = Symbols.AAPL;
var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var fileProvider = TestGlobals.DataProvider;
using var cache = new ZipDataCacheProvider(fileProvider);
var factory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, cache);
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(1998, 1, 1);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(false, null, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
enumerator.MoveNext();
}
date = date.AddDays(1);
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
public virtual void SetUp()
{
_algorithm = new QCAlgorithm();
_dataProvider = new DefaultDataProvider();
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(_dataProvider);
var factorFileProvider = new LocalZipFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, _dataProvider);
var historyProvider = new SubscriptionDataReaderHistoryProvider();
_cacheProvider = new ZipDataCacheProvider(_dataProvider);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null,
_dataProvider, _cacheProvider, mapFileProvider, factorFileProvider,
null, true, new DataPermissionManager()));
_algorithm.SetHistoryProvider(historyProvider);
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.Settings.FreePortfolioValue = 0;
}
public void CurrencyConversionRateResolved()
{
// Unit test to prove that in the event that default resolution (minute) history request returns
// no data for our currency conversion that BaseSetupHandler will use a daily history request
// to determine the the conversion rate if possible.
// Setup history provider and algorithm
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCache,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
var algorithm = new BrokerageSetupHandlerTests.TestAlgorithm {
UniverseSettings = { Resolution = Resolution.Minute }
};
algorithm.SetHistoryProvider(historyProvider);
// Pick a date range where we do NOT have BTCUSD minute data
algorithm.SetStartDate(2015, 1, 24);
algorithm.SetCash("USD", 0);
algorithm.SetCash("BTC", 10);
// Have BaseSetupHandler resolve the currency conversion
BaseSetupHandler.SetupCurrencyConversions(algorithm, algorithm.DataManager.UniverseSelection);
// Assert that our portfolio has some value and that value is bitcoin
Assert.IsTrue(algorithm.Portfolio.Cash > 0);
Assert.IsTrue(algorithm.Portfolio.CashBook["BTC"].ValueInAccountCurrency > 0);
zipCache.DisposeSafely();
}
public void EquitiesAreMappedCorrectly()
{
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
new DefaultDataProvider(),
zipCache,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
var symbol = Symbol.Create("WM", SecurityType.Equity, Market.USA);
var result = historyProvider.GetHistory(
new[]
{
new HistoryRequest(new DateTime(2008, 01, 01),
new DateTime(2008, 01, 05),
typeof(TradeBar),
symbol,
Resolution.Daily,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
TickType.Trade)
},
TimeZones.NewYork).ToList();
var firstBar = result.First().Values.Single();
Assert.AreEqual("WMI", firstBar.Symbol.Value);
Assert.IsNotEmpty(result);
zipCache.DisposeSafely();
}
public void ReadsZipEntryNames()
{
var time = new DateTime(2016, 03, 03, 12, 48, 15);
var source = Path.Combine("TestData", "20151224_quote_american.zip");
var config = new SubscriptionDataConfig(typeof(ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
TimeZones.NewYork, TimeZones.NewYork, false, false, false);
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, time, false);
var expected = new[]
{
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 22m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Put, 37m, new DateTime(2016, 08, 19)),
};
var actual = factory.Read(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)).ToList();
// we only really care about the symbols
CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol));
Assert.IsTrue(actual.All(x => x is ZipEntryName));
}
