public void CryptoPaths_CanBeParsedCorrectly() { DateTime date; Symbol symbol; Resolution resolution; var cryptoPath = "Data\\crypto\\gdax\\daily\\btcusd_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto); Assert.AreEqual(symbol.ID.Market, Market.GDAX); Assert.AreEqual(resolution, Resolution.Daily); Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd"); cryptoPath = "Data\\crypto\\gdax\\hour\\btcusd_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto); Assert.AreEqual(symbol.ID.Market, Market.GDAX); Assert.AreEqual(resolution, Resolution.Hour); Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd"); cryptoPath = "Data\\crypto\\gdax\\minute\\btcusd\\20161007_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(cryptoPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Crypto); Assert.AreEqual(symbol.ID.Market, Market.GDAX); Assert.AreEqual(resolution, Resolution.Minute); Assert.AreEqual(symbol.ID.Symbol.ToLowerInvariant(), "btcusd"); Assert.AreEqual(date.Date, Parse.DateTime("2016-10-07").Date); }
public void IncorrectPaths_CannotBeParsed() { DateTime date; Symbol symbol; Resolution resolution; var invalidPath = "forex/fxcm/eurusd/20160101_quote.zip"; Assert.IsFalse(LeanData.TryParsePath(invalidPath, out symbol, out date, out resolution)); var nonExistantPath = "Data/f/fxcm/eurusd/20160101_quote.zip"; Assert.IsFalse(LeanData.TryParsePath(nonExistantPath, out symbol, out date, out resolution)); var notAPath = "ooooooooooooooooooooooooooooooooooooooooooooooooooooooo"; Assert.IsFalse(LeanData.TryParsePath(notAPath, out symbol, out date, out resolution)); var emptyPath = ""; Assert.IsFalse(LeanData.TryParsePath(emptyPath, out symbol, out date, out resolution)); string nullPath = null; Assert.IsFalse(LeanData.TryParsePath(nullPath, out symbol, out date, out resolution)); var optionsTradePath = "Data/option/u sa/minute/aapl/20140606_trade_american.zip"; Assert.IsFalse(LeanData.TryParsePath(optionsTradePath, out symbol, out date, out resolution)); }
/// <summary> /// Get a lean data reader for a specific symbol to read the ticks /// </summary> /// <param name="symbol">The symbol being read</param> /// <param name="tickDateFile">The path to the tick file</param> /// <returns>A <see cref="LeanDataReader"/></returns> private static LeanDataReader GetLeanDataTickReader(Symbol symbol, TickType type, string tickDateFile) { Symbol sym; DateTime date; Resolution res; var subscription = new SubscriptionDataConfig(typeof(Tick), symbol, Resolution.Tick, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, false, type); LeanData.TryParsePath(tickDateFile, out sym, out date, out res); return(new LeanDataReader(subscription, symbol, Resolution.Tick, date, Globals.DataFolder)); }
public void AlternativePaths_CanBeParsedCorrectly(string path, string expectedSymbol, string expectedDate) { DateTime date; Symbol symbol; Resolution resolution; Assert.IsTrue(LeanData.TryParsePath(path, out symbol, out date, out resolution)); Assert.AreEqual(SecurityType.Base, symbol.SecurityType); Assert.AreEqual(Market.USA, symbol.ID.Market); Assert.AreEqual(Resolution.Daily, resolution); Assert.AreEqual(expectedSymbol, symbol.ID.Symbol.ToLowerInvariant()); Assert.AreEqual(expectedDate == null ? default(DateTime) : Parse.DateTime(expectedDate).Date, date); }
/// <summary> /// Determines if it should downloads new data and retrieves data from disc /// </summary> /// <param name="key">A string representing where the data is stored</param> /// <returns>A <see cref="Stream"/> of the data requested</returns> public override Stream Fetch(string key) { return(DownloadOnce(key, s => { if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType)) { var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); DateTime startTimeUtc; DateTime endTimeUtc; if (resolution < Resolution.Hour) { // we can get the date from the path startTimeUtc = date.ConvertToUtc(dataTimeZone); endTimeUtc = date.Add(resolution.ToTimeSpan()).ConvertToUtc(dataTimeZone); } else { try { startTimeUtc = symbol.ID.Date; } catch (InvalidOperationException) { startTimeUtc = Time.BeginningOfTime; } endTimeUtc = DateTime.UtcNow; } // Save the data var writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType); try { var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType); var data = _dataDownloader.Get(getParams) .Where(baseData => symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType) // for canonical symbols, downloader will return data for all of the chain .GroupBy(baseData => baseData.Symbol); foreach (var dataPerSymbol in data) { writer.Write(dataPerSymbol); } } catch (Exception e) { Log.Error(e); } } }));
/// <summary> /// Initialize a instance of LeanDataReader from a path to a zipped data file. /// It also supports declaring the zip entry CSV file for options and futures. /// </summary> /// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param> /// <example> /// var dataReader = LeanDataReader("../relative/path/to/file.zip") /// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv") /// </example> public LeanDataReader(string filepath) { Symbol symbol; DateTime date; Resolution resolution; string zipEntry = null; var isFutureOrOption = filepath.Contains("#"); if (isFutureOrOption) { zipEntry = filepath.Split('#')[1]; filepath = filepath.Split('#')[0]; } var fileInfo = new FileInfo(filepath); if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution)) { throw new ArgumentException($"File {filepath} cannot be parsed."); } if (isFutureOrOption) { symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry); } var marketHoursDataBase = MarketHoursDatabase.FromDataFolder(); var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; var tickType = LeanData.GetCommonTickType(symbol.SecurityType); var fileName = Path.GetFileNameWithoutExtension(fileInfo.Name); if (fileName.Contains("_")) { tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true); } var dataType = LeanData.GetDataType(resolution, tickType); var config = new SubscriptionDataConfig(dataType, symbol, resolution, dataTimeZone, exchangeTimeZone, tickType: tickType, fillForward: false, extendedHours: true, isInternalFeed: true); _date = date; _zipPath = fileInfo.FullName; _zipentry = zipEntry; _config = config; }
/// <summary> /// Retrieves data to be used in an algorithm. /// If file does not exist, an attempt is made to download them from the api /// </summary> /// <param name="key">A string representing where the data is stored</param> /// <returns>A <see cref="Stream"/> of the data requested</returns> public Stream Fetch(string key) { Symbol symbol; DateTime date; Resolution resolution; // Fetch the details of this data request if (LeanData.TryParsePath(key, out symbol, out date, out resolution)) { if (!File.Exists(key) || IsOutOfDate(resolution, key)) { return(DownloadData(key, symbol, date, resolution)); } // Use the file already on the disk return(new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read)); } Log.Error("ApiDataProvider.Fetch(): failed to parse key {0}", key); return(null); }
/// <summary> /// Retrieves data to be used in an algorithm. /// If file does not exist, an attempt is made to download them from the api /// </summary> /// <param name="key">A string representing where the data is stored</param> /// <returns>A <see cref="Stream"/> of the data requested</returns> public Stream Fetch(string key) { if (File.Exists(key)) { return(new FileStream(key, FileMode.Open, FileAccess.Read)); } // If the file cannot be found on disc, attempt to retrieve it from the API Symbol symbol; DateTime date; Resolution resolution; if (LeanData.TryParsePath(key, out symbol, out date, out resolution)) { Log.Trace("ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for symbol({0}), resolution({1}) and date({2}).", symbol.Value, resolution, date.Date.ToShortDateString()); var downloadSuccessful = _api.DownloadData(symbol, resolution, date); if (downloadSuccessful) { Log.Trace("ApiDataProvider.Fetch(): Successfully retrieved data for symbol({0}), resolution({1}) and date({2}).", symbol.Value, resolution, date.Date.ToShortDateString()); return(new FileStream(key, FileMode.Open, FileAccess.Read)); } } Log.Error("ApiDataProvider.Fetch(): Unable to remotely retrieve data for path {0}. " + "Please make sure you have the necessary data in your online QuantConnect data library.", key); return(null); }
public void CorrectPaths_CanBeParsedCorrectly() { DateTime date; Symbol symbol; Resolution resolution; var customPath = "a/very/custom/path/forex/oanda/tick/eurusd/20170104_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(customPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Forex); Assert.AreEqual(symbol.ID.Market, Market.Oanda); Assert.AreEqual(resolution, Resolution.Tick); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd"); Assert.AreEqual(date.Date, DateTime.Parse("2017-01-04").Date); var mixedPathSeperators = @"Data//forex/fxcm\/minute//eurusd\\20160101_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(mixedPathSeperators, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Forex); Assert.AreEqual(symbol.ID.Market, Market.FXCM); Assert.AreEqual(resolution, Resolution.Minute); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd"); Assert.AreEqual(date.Date, DateTime.Parse("2016-01-01").Date); var longRelativePath = "../../../../../../../../../Data/forex/fxcm/hour/gbpusd.zip"; Assert.IsTrue(LeanData.TryParsePath(longRelativePath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Forex); Assert.AreEqual(symbol.ID.Market, Market.FXCM); Assert.AreEqual(resolution, Resolution.Hour); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "gbpusd"); Assert.AreEqual(date.Date, DateTime.MinValue); var shortRelativePath = "Data/forex/fxcm/minute/eurusd/20160102_quote.zip"; Assert.IsTrue(LeanData.TryParsePath(shortRelativePath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Forex); Assert.AreEqual(symbol.ID.Market, Market.FXCM); Assert.AreEqual(resolution, Resolution.Minute); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "eurusd"); Assert.AreEqual(date.Date, DateTime.Parse("2016-01-02").Date); var dailyEquitiesPath = "Data/equity/usa/daily/aapl.zip"; Assert.IsTrue(LeanData.TryParsePath(dailyEquitiesPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Equity); Assert.AreEqual(symbol.ID.Market, Market.USA); Assert.AreEqual(resolution, Resolution.Daily); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "aapl"); Assert.AreEqual(date.Date, DateTime.MinValue); var minuteEquitiesPath = "Data/equity/usa/minute/googl/20070103_trade.zip"; Assert.IsTrue(LeanData.TryParsePath(minuteEquitiesPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Equity); Assert.AreEqual(symbol.ID.Market, Market.USA); Assert.AreEqual(resolution, Resolution.Minute); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "goog"); Assert.AreEqual(date.Date, DateTime.Parse("2007-01-03").Date); var cfdPath = "Data/cfd/oanda/minute/bcousd/20160101_trade.zip"; Assert.IsTrue(LeanData.TryParsePath(cfdPath, out symbol, out date, out resolution)); Assert.AreEqual(symbol.SecurityType, SecurityType.Cfd); Assert.AreEqual(symbol.ID.Market, Market.Oanda); Assert.AreEqual(resolution, Resolution.Minute); Assert.AreEqual(symbol.ID.Symbol.ToLower(), "bcousd"); Assert.AreEqual(date.Date, DateTime.Parse("2016-01-01").Date); }