/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data type to use.
/// If the config already existed will return existing instance instead
/// </summary>
public SubscriptionDataConfig Add(
Type dataType,
Symbol symbol,
Resolution resolution,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false
)
{
return(Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))
})
.First());
}
/// <summary>
/// Write the factor file to the correct place in the default Data folder
/// </summary>
/// <param name="symbol">The symbol this factor file represents</param>
public void WriteToCsv(Symbol symbol)
{
var filePath = LeanData.GenerateRelativeFactorFilePath(symbol);
var csv = new StringBuilder();
foreach (var kvp in SortedFactorFileData)
{
var newLine = string.Format("{0:yyyyMMdd},{1},{2}",
kvp.Key,
kvp.Value.PriceFactor,
kvp.Value.SplitFactor);
csv.AppendLine(newLine);
}
File.WriteAllText(filePath, csv.ToString());
}
/// <summary>
/// Method to download and save the data purchased through QuantConnect
/// </summary>
/// <param name="symbol">Symbol of security of which data will be requested.</param>
/// <param name="resolution">Resolution of data requested.</param>
/// <param name="date">Date of the data requested.</param>
/// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns>
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
// Get a link to the data
var link = ReadDataLink(symbol, resolution, date);
// Make sure the link was successfully retrieved
if (!link.Success)
{
return(false);
}
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
// Download and save the data
var uri = new Uri(link.DataLink);
var client = new RestClient(uri.Scheme + "://" + uri.Host);
var request = new RestRequest(uri.PathAndQuery, Method.GET);
// MAke a request for the data at the link
var response = client.Execute(request);
// If the response is JSON it doesn't contain any data, try and extract the message and write it
if (response.ContentType.ToLowerInvariant() == "application/json")
{
try
{
var contentObj = JObject.Parse(response.Content);
var message = contentObj["message"].Value <string>();
Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}");
}
catch
{
Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}. Api response could not be parsed.");
}
return(false);
}
// Any other case save the content to given path
response.RawBytes.SaveAs(path);
return(true);
}
private HistoryRequest CreateHistoryRequest(SubscriptionDataConfig subscription, DateTime startAlgoTz, DateTime endAlgoTz, Resolution?resolution)
{
resolution = resolution ?? subscription.Resolution;
// find the correct data type for the history request
var dataType = subscription.IsCustomData ? subscription.Type : LeanData.GetDataType(resolution.Value, subscription.TickType);
var request = new HistoryRequest(subscription, GetExchangeHours(subscription.Symbol), startAlgoTz.ConvertToUtc(TimeZone), endAlgoTz.ConvertToUtc(TimeZone))
{
DataType = dataType,
Resolution = resolution.Value,
FillForwardResolution = subscription.FillDataForward ? resolution : null,
TickType = subscription.TickType
};
return(request);
}
/// <summary>
/// Get source for tick data feed - not used with QuantConnect data sources implementation.
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String source location of the file to be opened with a stream</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
// Currently ticks aren't sourced through GetSource in live mode
return(new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile));
}
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
if (config.SecurityType == SecurityType.Option ||
config.SecurityType == SecurityType.Future)
{
source += "#" + LeanData.GenerateZipEntryName(config.Symbol, date, config.Resolution, config.TickType);
}
return(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv));
}
public void FactorFiles_CanBeGenerated_Accurately()
{
// Arrange
var yahooEvents = _yahooDataDownloader.DownloadSplitAndDividendData(_symbol, Parse.DateTime("01/01/1970"), DateTime.MaxValue);
var filePath = LeanData.GenerateRelativeFactorFilePath(_symbol);
var tolerance = 0.00001m;
if (!File.Exists(filePath))
{
throw new ArgumentException("This test requires an already calculated factor file." +
"Try using one of the pre-existing factor files ");
}
var originalFactorFileInstance = FactorFile.Read(PermTick, Market);
// Act
var newFactorFileInstance = _factorFileGenerator.CreateFactorFile(yahooEvents.ToList());
var earliestDate = originalFactorFileInstance.SortedFactorFileData.First().Key;
var latestDate = originalFactorFileInstance.SortedFactorFileData.Last().Key;
// Assert
Assert.AreEqual(originalFactorFileInstance.SortedFactorFileData.Count,
newFactorFileInstance.SortedFactorFileData.Count);
for (var i = earliestDate; i < latestDate; i = i.AddDays(1))
{
FactorFileRow expected = null;
FactorFileRow actual = null;
originalFactorFileInstance.SortedFactorFileData.TryGetValue(i, out expected);
newFactorFileInstance.SortedFactorFileData.TryGetValue(i, out actual);
if (expected == null || actual == null)
{
Assert.IsTrue(actual == null);
Assert.IsTrue(expected == null);
}
else
{
Assert.IsTrue(Math.Abs(expected.PriceFactor - actual.PriceFactor) < tolerance);
Assert.IsTrue(Math.Abs(expected.SplitFactor - actual.SplitFactor) < tolerance);
}
}
}
/// <summary>
/// Get the last known price using the history provider.
/// Useful for seeding securities with the correct price
/// </summary>
/// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
/// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
public BaseData GetLastKnownPrice(Security security)
{
if (security.Symbol.IsCanonical())
{
return(null);
}
// For speed and memory usage, use Resolution.Minute as the minimum resolution
var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)security.Resolution);
var startTime = GetStartTimeAlgoTzForSecurity(security, 1, resolution);
var endTime = Time.RoundDown(resolution.ToTimeSpan());
// Get the config with the largest resolution
var subscriptionDataConfig = GetMatchingSubscription(security, typeof(BaseData));
// if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar
if (subscriptionDataConfig != null && subscriptionDataConfig.Resolution == Resolution.Tick)
{
var dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType);
subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution);
}
var request = new HistoryRequest
{
StartTimeUtc = startTime.ConvertToUtc(_localTimeKeeper.TimeZone),
EndTimeUtc = endTime.ConvertToUtc(_localTimeKeeper.TimeZone),
DataType = subscriptionDataConfig != null ? subscriptionDataConfig.Type : typeof(TradeBar),
Resolution = resolution,
FillForwardResolution = resolution,
Symbol = security.Symbol,
ExchangeHours = security.Exchange.Hours
};
var history = History(new List <HistoryRequest> {
request
});
if (history.Any() && history.First().Values.Any())
{
return(history.First().Values.First());
}
return(null);
}
/// <summary>
/// Returns history requirements for the volatility model expressed in the form of history request
/// </summary>
/// <param name="security">The security of the request</param>
/// <param name="utcTime">The date/time of the request</param>
/// <returns>History request object list, or empty if no requirements</returns>
public override IEnumerable <HistoryRequest> GetHistoryRequirements(Security security, DateTime utcTime)
{
if (SubscriptionDataConfigProvider == null)
{
throw new InvalidOperationException(
"RelativeStandardDeviationVolatilityModel.GetHistoryRequirements(): " +
"SubscriptionDataConfigProvider was not set."
);
}
var configurations = SubscriptionDataConfigProvider
.GetSubscriptionDataConfigs(security.Symbol)
.ToList();
var barCount = _window.Size + 1;
// hour resolution does no have extended market hours data
var extendedMarketHours = _periodSpan != Time.OneHour && configurations.IsExtendedMarketHours();
var configuration = configurations.First();
var localStartTime = Time.GetStartTimeForTradeBars(
security.Exchange.Hours,
utcTime.ConvertFromUtc(security.Exchange.TimeZone),
_periodSpan,
barCount,
extendedMarketHours,
configuration.DataTimeZone);
var utcStartTime = localStartTime.ConvertToUtc(security.Exchange.TimeZone);
return(new[]
{
new HistoryRequest(utcStartTime,
utcTime,
typeof(TradeBar),
configuration.Symbol,
configurations.GetHighestResolution(),
security.Exchange.Hours,
configuration.DataTimeZone,
configurations.GetHighestResolution(),
extendedMarketHours,
configurations.IsCustomData(),
configurations.DataNormalizationMode(),
LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), security.Type))
});
}
public void FactorFiles_CanBeGenerated_Accurately()
{
// Arrange
var yahooEvents = _yahooDataDownloader.DownloadSplitAndDividendData(_symbol, Parse.DateTime("01/01/1970"), DateTime.MaxValue);
var filePath = LeanData.GenerateRelativeFactorFilePath(_symbol);
var tolerance = 0.00001m;
if (!File.Exists(filePath))
{
throw new ArgumentException("This test requires an already calculated factor file." +
"Try using one of the pre-existing factor files ");
}
var originalFactorFileInstance = TestGlobals.FactorFileProvider.Get(_symbol) as CorporateFactorProvider;
// we limit events to the penultimate time in our factor file (last one is 2050)
var lastValidRow = originalFactorFileInstance.SortedFactorFileData.Reverse().Skip(1).First();
// Act
var newFactorFileInstance = _factorFileGenerator.CreateFactorFile(yahooEvents.Where(data => data.Time.AddDays(-1) <= lastValidRow.Key).ToList());
var earliestDate = originalFactorFileInstance.SortedFactorFileData.First().Key;
var latestDate = originalFactorFileInstance.SortedFactorFileData.Last().Key;
// Assert
Assert.AreEqual(originalFactorFileInstance.SortedFactorFileData.Count,
newFactorFileInstance.SortedFactorFileData.Count);
for (var i = earliestDate; i < latestDate; i = i.AddDays(1))
{
originalFactorFileInstance.SortedFactorFileData.TryGetValue(i, out var expected);
newFactorFileInstance.SortedFactorFileData.TryGetValue(i, out var actual);
if (expected == null || actual == null)
{
Assert.IsTrue(actual == null);
Assert.IsTrue(expected == null);
}
else
{
Assert.IsTrue(Math.Abs(expected.Single().PriceFactor - actual.Single().PriceFactor) < tolerance);
Assert.IsTrue(Math.Abs(expected.Single().SplitFactor - actual.Single().SplitFactor) < tolerance);
}
}
}
private static ISubscriptionDataSourceReader Initialize(bool liveMode, Resolution resolution, out SubscriptionDataSource source)
{
using var dataProvider = new DefaultDataProvider();
using var cache = new ZipDataCacheProvider(dataProvider);
var config = new SubscriptionDataConfig(typeof(TestBaseDataCollection),
Symbols.SPY,
resolution,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false);
var date = DateTime.MinValue;
var path = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, resolution, TickType.Trade);
source = new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile);
return(new BaseDataCollectionAggregatorReader(cache, config, date, liveMode));
}
private static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
{
var now = DateTime.UtcNow;
var dataType = LeanData.GetDataType(resolution, tickType);
return(new HistoryRequest(now.Add(-period),
now,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
true,
false,
DataNormalizationMode.Adjusted,
tickType));
}
public void OptionZipFilePathWithUnderlyingEquity()
{
var underlying = Symbol.Create("SPY", SecurityType.Equity, QuantConnect.Market.USA);
var optionSymbol = Symbol.CreateOption(
underlying,
Market.USA,
OptionStyle.American,
OptionRight.Put,
4200m,
new DateTime(2020, 12, 31));
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote);
Assert.AreEqual("../../../Data/option/usa/minute/spy/20200922_quote_american.zip", optionZipFilePath);
Assert.AreEqual("20200922_spy_minute_quote_american_put_42000000_20201231.csv", optionEntryFilePath);
}
/// <summary>
/// Will determine if price scaling should be used for this subscription configuration
/// </summary>
/// <param name="config">The subscription data configuration we are processing</param>
/// <remarks>One of the objectives of this method is to normalize the 'use price scale'
/// check and void code duplication and related issues</remarks>
/// <param name="liveMode">True, is this is a live mode data stream</param>
/// <returns>True if ticker prices should be scaled</returns>
public static bool PricesShouldBeScaled(this SubscriptionDataConfig config, bool liveMode = false)
{
if (config.IsCustomData || config.Symbol.Value.Contains("UNIVERSE"))
{
return(false);
}
if (config.SecurityType == SecurityType.Equity && !liveMode)
{
return(true);
}
if (config.SecurityType == SecurityType.Future && config.Symbol.IsCanonical())
{
return(LeanData.IsCommonLeanDataType(config.Type));
}
return(false);
}
public void OptionZipFilePathWithUnderlyingFuture(string futureOptionTicker, string expectedFutureOptionTicker)
{
var underlying = Symbol.CreateFuture(futureOptionTicker, Market.CME, new DateTime(2021, 3, 19));
var optionSymbol = Symbol.CreateOption(
underlying,
Market.CME,
OptionStyle.American,
OptionRight.Put,
4200m,
new DateTime(2021, 3, 18));
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote);
Assert.AreEqual($"../../../Data/futureoption/cme/minute/{expectedFutureOptionTicker.ToLowerInvariant()}/{underlying.ID.Date:yyyyMMdd}/20200922_quote_american.zip", optionZipFilePath);
Assert.AreEqual($"20200922_{expectedFutureOptionTicker.ToLowerInvariant()}_minute_quote_american_put_42000000_{optionSymbol.ID.Date:yyyyMMdd}.csv", optionEntryFilePath);
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="symbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
{
if (underlyingSymbol.SecurityType != SecurityType.Equity && underlyingSymbol.SecurityType != SecurityType.Future)
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity or SecurityType.Future is expected but was {underlyingSymbol.SecurityType}");
}
// build the option contract list from the open interest zip file entry names
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(underlyingSymbol, underlyingSymbol.ID.Market, default(OptionStyle), default(OptionRight), 0, SecurityIdentifier.DefaultDate);
var fileExists = false;
var zipFileName = string.Empty;
// In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed
// to have the complete option chain. Quotes come after open-interest
// because it's also likely to contain the option chain. Trades may be
// missing portions of the option chain, so we resort to it last.
foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade })
{
// build the zip file name for open interest data
zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
if (File.Exists(zipFileName))
{
fileExists = true;
break;
}
}
if (!fileExists)
{
Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
yield break;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
}
public void LeanDataWriter_CanWriteZipWithMultipleContracts(SecurityType securityType)
{
Symbol contract1;
Symbol contract2;
if (securityType == SecurityType.Future)
{
contract1 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 03, 01));
}
else if (securityType == SecurityType.Option)
{
contract1 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
}
else if (securityType == SecurityType.FutureOption)
{
contract1 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
}
else
{
throw new NotImplementedException($"{securityType} not implemented!");
}
var filePath1 = LeanData.GenerateZipFilePath(_dataDirectory, contract1, _date, Resolution.Second, TickType.Quote);
var leanDataWriter1 = new LeanDataWriter(Resolution.Second, contract1, _dataDirectory, TickType.Quote);
leanDataWriter1.Write(GetQuoteBars(contract1));
var filePath2 = LeanData.GenerateZipFilePath(_dataDirectory, contract2, _date, Resolution.Second, TickType.Quote);
var leanDataWriter2 = new LeanDataWriter(Resolution.Second, contract2, _dataDirectory, TickType.Quote);
leanDataWriter2.Write(GetQuoteBars(contract2));
Assert.AreEqual(filePath1, filePath2);
Assert.IsTrue(File.Exists(filePath1));
Assert.IsFalse(File.Exists(filePath1 + ".tmp"));
var data = QuantConnect.Compression.Unzip(filePath1).ToDictionary(x => x.Key, x => x.Value.ToList());
Assert.AreEqual(2, data.Count);
Assert.That(data.Values, Has.All.Count.EqualTo(3));
}
public void FutureOptionSingleZipFileContainingMultipleFuturesOptionsContracts(OptionRight right, int strike, int year, int month, int day)
{
var underlying = Symbol.CreateFuture("GC", Market.COMEX, new DateTime(2020, 4, 28));
var expiry = new DateTime(year, month, day);
var optionSymbol = Symbol.CreateOption(
underlying,
Market.COMEX,
OptionStyle.American,
right,
(decimal)strike,
expiry);
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote);
Assert.AreEqual("../../../Data/futureoption/comex/minute/og/20200428/20200105_quote_american.zip", optionZipFilePath);
Assert.AreEqual($"20200105_og_minute_quote_american_{right.ToLower()}_{strike}0000_{expiry:yyyyMMdd}.csv", optionEntryFilePath);
}
public void GetDataType_ReturnsCorrectType()
{
var tickType = typeof(Tick);
var openInterestType = typeof(OpenInterest);
var quoteBarType = typeof(QuoteBar);
var tradeBarType = typeof(TradeBar);
Assert.AreEqual(LeanData.GetDataType(Resolution.Tick, TickType.OpenInterest), tickType);
Assert.AreNotEqual(LeanData.GetDataType(Resolution.Daily, TickType.OpenInterest), tickType);
Assert.AreEqual(LeanData.GetDataType(Resolution.Second, TickType.OpenInterest), openInterestType);
Assert.AreNotEqual(LeanData.GetDataType(Resolution.Tick, TickType.OpenInterest), openInterestType);
Assert.AreEqual(LeanData.GetDataType(Resolution.Minute, TickType.Quote), quoteBarType);
Assert.AreNotEqual(LeanData.GetDataType(Resolution.Second, TickType.Trade), quoteBarType);
Assert.AreEqual(LeanData.GetDataType(Resolution.Hour, TickType.Trade), tradeBarType);
Assert.AreNotEqual(LeanData.GetDataType(Resolution.Tick, TickType.OpenInterest), tradeBarType);
}
/// <summary>
/// Creates the <see cref="TextWriter"/> that writes data to csv files
/// </summary>
private Writer CreateTextWriter(BaseData data)
{
var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType);
var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType)
.Replace(".zip", string.Empty);
var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry);
var directory = new FileInfo(path).Directory.FullName;
if (!Directory.Exists(directory))
{
// lock before checking again
lock (DirectoryCreateSync) if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
}
return(new Writer(path, new StreamWriter(path)));
}
public void HandlesOpenInterestTicks([ValueSource(nameof(ResolutionCases))] Resolution resolution, [ValueSource(nameof(SymbolCases))] Symbol symbol)
{
// Arrange
var converter = new PandasConverter();
var tickType = TickType.OpenInterest;
var dataType = LeanData.GetDataType(resolution, tickType);
var subcriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, resolution,
TimeZones.Chicago, TimeZones.Chicago,
tickType: tickType, fillForward: false,
extendedHours: true, isInternalFeed: true);
var openinterest = new List <OpenInterest>();
for (int i = 0; i < 10; i++)
{
var line = $"{1000 * i},{11 * i}";
var openInterestTicks = new OpenInterest(subcriptionDataConfig, symbol, line, new DateTime(2017, 10, 10));
openinterest.Add(openInterestTicks);
}
// Act
dynamic dataFrame = converter.GetDataFrame(openinterest);
//Assert
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol.Value];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("openinterest") != null);
var count = subDataFrame.shape[0].AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].openinterest.AsManagedObject(typeof(decimal));
Assert.AreEqual(openinterest[i].Value, value);
}
}
}
/// <summary>
/// Fetch data from the cache
/// </summary>
/// <param name="key">A string representing the key of the cached data</param>
/// <returns>An <see cref="Stream"/> of the cached data</returns>
public Stream Fetch(string key)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
if (!File.Exists(filePath))
{
return(null);
}
try
{
using (var zip = ZipFile.Read(filePath))
{
ZipEntry entry;
if (entryName.IsNullOrEmpty())
{
// Return the first entry
entry = zip[0];
}
else
{
// Attempt to find our specific entry
if (!zip.ContainsEntry(entryName))
{
return(null);
}
entry = zip[entryName];
}
// Extract our entry and return it
var stream = new MemoryStream();
entry.Extract(stream);
stream.Position = 0;
return(stream);
}
}
catch (ZipException exception)
{
Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
return(null);
}
}
/// <summary>
/// Creates a new history request
/// </summary>
/// <param name="subscription">The config </param>
/// <param name="startAlgoTz">History request start time in algorithm time zone</param>
/// <param name="endAlgoTz">History request end time in algorithm time zone</param>
/// <param name="exchangeHours">Security exchange hours</param>
/// <param name="resolution">The resolution to use. If null will use <see cref="SubscriptionDataConfig.Resolution"/></param>
/// <param name="dataMappingMode">The contract mapping mode to use for the security history request</param>
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <returns>The new <see cref="HistoryRequest"/></returns>
public HistoryRequest CreateHistoryRequest(SubscriptionDataConfig subscription,
DateTime startAlgoTz,
DateTime endAlgoTz,
SecurityExchangeHours exchangeHours,
Resolution?resolution,
DataMappingMode?dataMappingMode = null,
DataNormalizationMode?dataNormalizationMode = null)
{
resolution ??= subscription.Resolution;
var dataType = subscription.Type;
// if we change resolution the data type can change, for example subscription being Tick type and resolution daily
// data type here won't be Tick anymore, but TradeBar/QuoteBar
if (resolution.Value != subscription.Resolution && LeanData.IsCommonLeanDataType(dataType))
{
dataType = LeanData.GetDataType(resolution.Value, subscription.TickType);
}
var request = new HistoryRequest(subscription,
exchangeHours,
startAlgoTz.ConvertToUtc(_algorithm.TimeZone),
endAlgoTz.ConvertToUtc(_algorithm.TimeZone))
{
DataType = dataType,
Resolution = resolution.Value,
FillForwardResolution = subscription.FillDataForward ? resolution : null,
TickType = subscription.TickType
};
if (dataMappingMode != null)
{
request.DataMappingMode = dataMappingMode.Value;
}
if (dataNormalizationMode != null)
{
request.DataNormalizationMode = dataNormalizationMode.Value;
}
return(request);
}
/// <summary>
/// Get the data feed types for a given <see cref="SecurityType"/> <see cref="Resolution"/>
/// </summary>
/// <param name="symbolSecurityType">The <see cref="SecurityType"/> used to determine the types</param>
/// <param name="resolution">The resolution of the data requested</param>
/// <param name="isCanonical">Indicates whether the security is Canonical (future and options)</param>
/// <returns>Types that should be added to the <see cref="SubscriptionDataConfig"/></returns>
public List <Type> LookupSubscriptionConfigDataTypes(SecurityType symbolSecurityType, Resolution resolution, bool isCanonical)
{
if (isCanonical)
{
return(new List <Type>()
{
typeof(ZipEntryName)
});
}
if (resolution == Resolution.Tick)
{
return(new List <Type>()
{
typeof(Tick)
});
}
return(AvailableDataTypes[symbolSecurityType].Select(tickType => LeanData.GetDataType(resolution, tickType)).ToList());
}
/// <summary>
/// Invoked for each piece of data from the source file
/// </summary>
/// <param name="data">The data to be processed</param>
public void Process(BaseData data)
{
Writer writer;
if (!_writers.TryGetValue(data.Symbol, out writer))
{
writer = CreateTextWriter(data);
_writers[data.Symbol] = writer;
}
// flush every so often
if (++writer.ProcessCount % TicksPerFlush == 0)
{
writer.TextWriter.Flush();
}
var line = LeanData.GenerateLine(data, data.Symbol.ID.SecurityType, _resolution);
writer.TextWriter.WriteLine(line);
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (symbol.SecurityType != SecurityType.Equity &&
symbol.SecurityType != SecurityType.Forex &&
symbol.SecurityType != SecurityType.Crypto)
{
throw new NotSupportedException($"Security type not supported: {symbol.SecurityType}");
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var dataType = LeanData.GetDataType(resolution, tickType);
var historyRequest =
new HistoryRequest(startUtc,
endUtc,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
resolution,
true,
false,
DataNormalizationMode.Adjusted,
tickType);
foreach (var baseData in _historyProvider.GetHistory(historyRequest))
{
yield return(baseData);
}
}
/// <summary>
/// Uses the orders to load data
/// </summary>
/// <param name="configs">Configurations to use for reading data</param>
/// <param name="start">Starting date to read data for</param>
/// <param name="end">Ending date to read data for</param>
/// <returns>List of enumerators of data</returns>
private List <IEnumerator <BaseData> > ReadData(
IEnumerable <Order> orders,
HashSet <Symbol> orderSymbols,
IEnumerable <SubscriptionDataConfig> configs,
DateTime date)
{
var readers = new List <IEnumerator <BaseData> >();
var symbolsOnDate = new HashSet <Symbol>(orders
.Where(o => (o.LastFillTime ?? o.Time).Date == date)
.Select(o => o.Symbol));
// If the config is not in the order Symbols at all, then it's a currency conversion
// feed, and should be loaded on every day.
var configsOnDate = configs
.Where(c => symbolsOnDate.Contains(c.Symbol) || !orderSymbols.Contains(c.Symbol))
.ToList();
foreach (var config in configsOnDate)
{
var mappedSymbol = config.Symbol;
if (config.Symbol.SecurityType == SecurityType.Equity || config.Symbol.SecurityType == SecurityType.Option)
{
MapFile mapFile;
if (!_mapFileCache.TryGetValue(config.Symbol, out mapFile))
{
mapFile = _mapFileResolver.ResolveMapFile(config.Symbol, null);
_mapFileCache[config.Symbol] = mapFile;
}
mappedSymbol = config.Symbol.UpdateMappedSymbol(mapFile.GetMappedSymbol(date, config.Symbol.Value));
}
if (File.Exists(LeanData.GenerateZipFilePath(Globals.DataFolder, mappedSymbol, date, _resolution, config.TickType)))
{
readers.Add(new LeanDataReader(config, mappedSymbol, _resolution, date, Globals.DataFolder).Parse().GetEnumerator());
}
}
return(readers);
}
/// <summary>
/// Retrieves data to be used in an algorithm.
/// If file does not exist, an attempt is made to download them from the api
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public Stream Fetch(string key)
{
Symbol symbol;
DateTime date;
Resolution resolution;
// Fetch the details of this data request
if (LeanData.TryParsePath(key, out symbol, out date, out resolution))
{
if (!File.Exists(key) || IsOutOfDate(resolution, key))
{
return(DownloadData(key, symbol, date, resolution));
}
// Use the file already on the disk
return(new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read));
}
Log.Error("ApiDataProvider.Fetch(): failed to parse key {0}", key);
return(null);
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data type to use.
/// If the config already existed will return existing instance instead
/// </summary>
public SubscriptionDataConfig Add(
Type dataType,
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
uint contractDepthOffset = 0
)
{
return(Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))
},
dataNormalizationMode, dataMappingMode, contractDepthOffset)
.First());
}
/// <summary>
/// Initialize a instance of LeanDataReader from a path to a zipped data file.
/// It also supports declaring the zip entry CSV file for options and futures.
/// </summary>
/// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param>
/// <example>
/// var dataReader = LeanDataReader("../relative/path/to/file.zip")
/// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv")
/// </example>
public LeanDataReader(string filepath)
{
Symbol symbol;
DateTime date;
Resolution resolution;
string zipEntry = null;
var isFutureOrOption = filepath.Contains("#");
if (isFutureOrOption)
{
zipEntry = filepath.Split('#')[1];
filepath = filepath.Split('#')[0];
}
var fileInfo = new FileInfo(filepath);
if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution, out var tickType, out var dataType))
{
throw new ArgumentException($"File {filepath} cannot be parsed.");
}
if (isFutureOrOption)
{
symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry);
}
var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
var config = new SubscriptionDataConfig(dataType, symbol, resolution,
dataTimeZone, exchangeTimeZone, tickType: tickType,
fillForward: false, extendedHours: true, isInternalFeed: true);
_date = date;
_zipPath = fileInfo.FullName;
_zipentry = zipEntry;
_config = config;
}
/// <summary>
/// Creates a security and matching configuration. This applies the default leverage if
/// leverage is less than or equal to zero.
/// This method also add the new symbol mapping to the <see cref="SymbolCache"/>
/// </summary>
public static Security CreateSecurity(Type dataType,
SecurityPortfolioManager securityPortfolioManager,
SubscriptionManager subscriptionManager,
SecurityExchangeHours exchangeHours,
DateTimeZone dataTimeZone,
SymbolProperties symbolProperties,
ISecurityInitializer securityInitializer,
Symbol symbol,
Resolution resolution,
bool fillDataForward,
decimal leverage,
bool extendedMarketHours,
bool isInternalFeed,
bool isCustomData,
bool isLiveMode,
bool addToSymbolCache = true,
bool isFilteredSubscription = true)
{
return(CreateSecurity(
new List <Tuple <Type, TickType> >
{
new Tuple <Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))
},
securityPortfolioManager,
subscriptionManager,
exchangeHours,
dataTimeZone,
symbolProperties,
securityInitializer,
symbol,
resolution,
fillDataForward,
leverage,
extendedMarketHours,
isInternalFeed,
isCustomData,
isLiveMode,
addToSymbolCache,
isFilteredSubscription));
}
