public void CryptoPaths_CanBeParsedCorrectly()
{
DateTime date;
Symbol symbol;
Resolution resolution;
var cryptoPath = "Data\\crypto\\gdax\\daily\\btcusd_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto);
Assert.AreEqual(symbol.ID.Market, Market.GDAX);
Assert.AreEqual(resolution, Resolution.Daily);
Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd");
cryptoPath = "Data\\crypto\\gdax\\hour\\btcusd_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto);
Assert.AreEqual(symbol.ID.Market, Market.GDAX);
Assert.AreEqual(resolution, Resolution.Hour);
Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd");
cryptoPath = "Data\\crypto\\gdax\\minute\\btcusd\\20161007_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto);
Assert.AreEqual(symbol.ID.Market, Market.GDAX);
Assert.AreEqual(resolution, Resolution.Minute);
Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd");
Assert.AreEqual(date.Date, Parse.DateTime("2016-10-07").Date);
}
public void IncorrectPaths_CannotBeParsed()
{
DateTime date;
Symbol symbol;
Resolution resolution;
var invalidPath = "forex/fxcm/eurusd/20160101_quote.zip";
Assert.IsFalse(LeanData.TryParsePath(invalidPath, out symbol, out date, out resolution));
var nonExistantPath = "Data/f/fxcm/eurusd/20160101_quote.zip";
Assert.IsFalse(LeanData.TryParsePath(nonExistantPath, out symbol, out date, out resolution));
var notAPath = "ooooooooooooooooooooooooooooooooooooooooooooooooooooooo";
Assert.IsFalse(LeanData.TryParsePath(notAPath, out symbol, out date, out resolution));
var emptyPath = "";
Assert.IsFalse(LeanData.TryParsePath(emptyPath, out symbol, out date, out resolution));
string nullPath = null;
Assert.IsFalse(LeanData.TryParsePath(nullPath, out symbol, out date, out resolution));
var optionsTradePath = "Data/option/u sa/minute/aapl/20140606_trade_american.zip";
Assert.IsFalse(LeanData.TryParsePath(optionsTradePath, out symbol, out date, out resolution));
}
/// <summary>
/// Get a lean data reader for a specific symbol to read the ticks
/// </summary>
/// <param name="symbol">The symbol being read</param>
/// <param name="tickDateFile">The path to the tick file</param>
/// <returns>A <see cref="LeanDataReader"/></returns>
private static LeanDataReader GetLeanDataTickReader(Symbol symbol, TickType type, string tickDateFile)
{
Symbol sym;
DateTime date;
Resolution res;
var subscription = new SubscriptionDataConfig(typeof(Tick), symbol, Resolution.Tick,
DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, false, type);
LeanData.TryParsePath(tickDateFile, out sym, out date, out res);
return(new LeanDataReader(subscription, symbol, Resolution.Tick, date, Globals.DataFolder));
}
public void AlternativePaths_CanBeParsedCorrectly(string path, string expectedSymbol, string expectedDate)
{
DateTime date;
Symbol symbol;
Resolution resolution;
Assert.IsTrue(LeanData.TryParsePath(path, out symbol, out date, out resolution));
Assert.AreEqual(SecurityType.Base, symbol.SecurityType);
Assert.AreEqual(Market.USA, symbol.ID.Market);
Assert.AreEqual(Resolution.Daily, resolution);
Assert.AreEqual(expectedSymbol, symbol.ID.Symbol.ToLowerInvariant());
Assert.AreEqual(expectedDate == null ? default(DateTime) : Parse.DateTime(expectedDate).Date, date);
}
/// <summary>
/// Determines if it should downloads new data and retrieves data from disc
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public override Stream Fetch(string key)
{
return(DownloadOnce(key, s =>
{
if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType))
{
var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
DateTime startTimeUtc;
DateTime endTimeUtc;
if (resolution < Resolution.Hour)
{
// we can get the date from the path
startTimeUtc = date.ConvertToUtc(dataTimeZone);
endTimeUtc = date.Add(resolution.ToTimeSpan()).ConvertToUtc(dataTimeZone);
}
else
{
try
{
startTimeUtc = symbol.ID.Date;
}
catch (InvalidOperationException)
{
startTimeUtc = Time.BeginningOfTime;
}
endTimeUtc = DateTime.UtcNow;
}
// Save the data
var writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType);
try
{
var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
var data = _dataDownloader.Get(getParams)
.Where(baseData => symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
// for canonical symbols, downloader will return data for all of the chain
.GroupBy(baseData => baseData.Symbol);
foreach (var dataPerSymbol in data)
{
writer.Write(dataPerSymbol);
}
}
catch (Exception e)
{
Log.Error(e);
}
}
}));
/// <summary>
/// Initialize a instance of LeanDataReader from a path to a zipped data file.
/// It also supports declaring the zip entry CSV file for options and futures.
/// </summary>
/// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param>
/// <example>
/// var dataReader = LeanDataReader("../relative/path/to/file.zip")
/// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv")
/// </example>
public LeanDataReader(string filepath)
{
Symbol symbol;
DateTime date;
Resolution resolution;
string zipEntry = null;
var isFutureOrOption = filepath.Contains("#");
if (isFutureOrOption)
{
zipEntry = filepath.Split('#')[1];
filepath = filepath.Split('#')[0];
}
var fileInfo = new FileInfo(filepath);
if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution))
{
throw new ArgumentException($"File {filepath} cannot be parsed.");
}
if (isFutureOrOption)
{
symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry);
}
var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
var tickType = LeanData.GetCommonTickType(symbol.SecurityType);
var fileName = Path.GetFileNameWithoutExtension(fileInfo.Name);
if (fileName.Contains("_"))
{
tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true);
}
var dataType = LeanData.GetDataType(resolution, tickType);
var config = new SubscriptionDataConfig(dataType, symbol, resolution,
dataTimeZone, exchangeTimeZone, tickType: tickType,
fillForward: false, extendedHours: true, isInternalFeed: true);
_date = date;
_zipPath = fileInfo.FullName;
_zipentry = zipEntry;
_config = config;
}
/// <summary>
/// Retrieves data to be used in an algorithm.
/// If file does not exist, an attempt is made to download them from the api
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public Stream Fetch(string key)
{
Symbol symbol;
DateTime date;
Resolution resolution;
// Fetch the details of this data request
if (LeanData.TryParsePath(key, out symbol, out date, out resolution))
{
if (!File.Exists(key) || IsOutOfDate(resolution, key))
{
return(DownloadData(key, symbol, date, resolution));
}
// Use the file already on the disk
return(new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read));
}
Log.Error("ApiDataProvider.Fetch(): failed to parse key {0}", key);
return(null);
}
/// <summary>
/// Retrieves data to be used in an algorithm.
/// If file does not exist, an attempt is made to download them from the api
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public Stream Fetch(string key)
{
if (File.Exists(key))
{
return(new FileStream(key, FileMode.Open, FileAccess.Read));
}
// If the file cannot be found on disc, attempt to retrieve it from the API
Symbol symbol;
DateTime date;
Resolution resolution;
if (LeanData.TryParsePath(key, out symbol, out date, out resolution))
{
Log.Trace("ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for symbol({0}), resolution({1}) and date({2}).",
symbol.Value,
resolution,
date.Date.ToShortDateString());
var downloadSuccessful = _api.DownloadData(symbol, resolution, date);
if (downloadSuccessful)
{
Log.Trace("ApiDataProvider.Fetch(): Successfully retrieved data for symbol({0}), resolution({1}) and date({2}).",
symbol.Value,
resolution,
date.Date.ToShortDateString());
return(new FileStream(key, FileMode.Open, FileAccess.Read));
}
}
Log.Error("ApiDataProvider.Fetch(): Unable to remotely retrieve data for path {0}. " +
"Please make sure you have the necessary data in your online QuantConnect data library.",
key);
return(null);
}
public void CorrectPaths_CanBeParsedCorrectly()
{
DateTime date;
Symbol symbol;
Resolution resolution;
var customPath = "a/very/custom/path/forex/oanda/tick/eurusd/20170104_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(customPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Forex);
Assert.AreEqual(symbol.ID.Market, Market.Oanda);
Assert.AreEqual(resolution, Resolution.Tick);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd");
Assert.AreEqual(date.Date, DateTime.Parse("2017-01-04").Date);
var mixedPathSeperators = @"Data//forex/fxcm\/minute//eurusd\\20160101_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(mixedPathSeperators, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Forex);
Assert.AreEqual(symbol.ID.Market, Market.FXCM);
Assert.AreEqual(resolution, Resolution.Minute);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd");
Assert.AreEqual(date.Date, DateTime.Parse("2016-01-01").Date);
var longRelativePath = "../../../../../../../../../Data/forex/fxcm/hour/gbpusd.zip";
Assert.IsTrue(LeanData.TryParsePath(longRelativePath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Forex);
Assert.AreEqual(symbol.ID.Market, Market.FXCM);
Assert.AreEqual(resolution, Resolution.Hour);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "gbpusd");
Assert.AreEqual(date.Date, DateTime.MinValue);
var shortRelativePath = "Data/forex/fxcm/minute/eurusd/20160102_quote.zip";
Assert.IsTrue(LeanData.TryParsePath(shortRelativePath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Forex);
Assert.AreEqual(symbol.ID.Market, Market.FXCM);
Assert.AreEqual(resolution, Resolution.Minute);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd");
Assert.AreEqual(date.Date, DateTime.Parse("2016-01-02").Date);
var dailyEquitiesPath = "Data/equity/usa/daily/aapl.zip";
Assert.IsTrue(LeanData.TryParsePath(dailyEquitiesPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Equity);
Assert.AreEqual(symbol.ID.Market, Market.USA);
Assert.AreEqual(resolution, Resolution.Daily);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "aapl");
Assert.AreEqual(date.Date, DateTime.MinValue);
var minuteEquitiesPath = "Data/equity/usa/minute/googl/20070103_trade.zip";
Assert.IsTrue(LeanData.TryParsePath(minuteEquitiesPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Equity);
Assert.AreEqual(symbol.ID.Market, Market.USA);
Assert.AreEqual(resolution, Resolution.Minute);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "goog");
Assert.AreEqual(date.Date, DateTime.Parse("2007-01-03").Date);
var cfdPath = "Data/cfd/oanda/minute/bcousd/20160101_trade.zip";
Assert.IsTrue(LeanData.TryParsePath(cfdPath, out symbol, out date, out resolution));
Assert.AreEqual(symbol.SecurityType, SecurityType.Cfd);
Assert.AreEqual(symbol.ID.Market, Market.Oanda);
Assert.AreEqual(resolution, Resolution.Minute);
Assert.AreEqual(symbol.ID.Symbol.ToLower(), "bcousd");
Assert.AreEqual(date.Date, DateTime.Parse("2016-01-01").Date);
}