/// <summary> /// Method to download and save the data purchased through QuantConnect /// </summary> /// <param name="symbol">Symbol of security of which data will be requested.</param> /// <param name="resolution">Resolution of data requested.</param> /// <param name="date">Date of the data requested.</param> /// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns> public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) { // Get a link to the data var link = ReadDataLink(symbol, resolution, date); // Make sure the link was successfully retrieved if (!link.Success) { return(false); } // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); // Download and save the data var uri = new Uri(link.DataLink); var client = new RestClient(uri.Scheme + "://" + uri.Host); var request = new RestRequest(uri.PathAndQuery, Method.GET); // If the response is not a zip then it is not data, don't write it. var response = client.Execute(request); if (response.ContentType != "application/zip") { var message = JObject.Parse(response.Content)["message"].Value <string>(); Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}"); return(false); } response.RawBytes.SaveAs(path); return(true); }
/// <summary> /// Method to download and save the data purchased through QuantConnect /// </summary> /// <param name="symbol">Symbol of security of which data will be requested.</param> /// <param name="resolution">Resolution of data requested.</param> /// <param name="date">Date of the data requested.</param> /// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns> public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) { // Get a link to the data var link = ReadDataLink(symbol, resolution, date); // Make sure the link was successfully retrieved if (!link.Success) { return(false); } // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); // Download and save the data var uri = new Uri(link.DataLink); var client = new RestClient(uri.Scheme + "://" + uri.Host); var request = new RestRequest(uri.PathAndQuery, Method.GET); client.DownloadData(request).SaveAs(path); return(true); }
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) { string r = "S5"; string filename = ""; switch (resolution) { case Resolution.Minute: r = "M1"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_minute_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol }); break; case Resolution.Second: r = "S5"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_second_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol }); break; case Resolution.Tick: r = "S5"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_tick_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol }); break; case Resolution.Hour: r = "H1"; filename = string.Format("{0}.csv", symbol.ID.Symbol); break; case Resolution.Daily: r = "D"; filename = string.Format("{0}.csv", symbol.ID.Symbol); break; } RestClient client = new RestClient(); var auth = "Bearer " + _token; client.Timeout = 120000; string s = string.Format("{0}_{1}", symbol.ID.Symbol.Substring(0, symbol.ID.Symbol.Length - 3), symbol.ID.Symbol.Substring(symbol.ID.Symbol.Length - 3)); DateTime todate = date.ToUniversalTime().AddDays(1); string url = string.Format(_url, s, _price, ToUnixTimestamp(date.ToUniversalTime()), ToUnixTimestamp(todate), r); client.BaseUrl = new Uri(url); Log.Trace(string.Format("Downloading {0}", url)); var request = new RestRequest(); request.AddHeader("content-type", "application/json"); request.AddHeader("Authorization", auth); IRestResponse response = client.Execute(request); string json = response.Content; // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); CreateZip(path, filename, JSonToCSV(date, json), json); return(true); }
private void SaveMinuteOrSecondOrTick( List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc, Symbol canonicalSymbol, IReadOnlyDictionary <Symbol, List <IGrouping <DateTime, BaseData> > > historyBySymbol) { var date = startTimeUtc; while (date <= endTimeUtc) { var zipFileName = Path.Combine( Globals.DataFolder, LeanData.GenerateRelativeZipFilePath(canonicalSymbol, date, _resolution, _tickType)); var folder = Path.GetDirectoryName(zipFileName); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } if (File.Exists(zipFileName)) { File.Delete(zipFileName); } using (var zip = new ZipFile(zipFileName)) { foreach (var symbol in symbols) { var zipEntryName = LeanData.GenerateZipEntryName(symbol, date, _resolution, _tickType); foreach (var group in historyBySymbol[symbol]) { if (group.Key == date) { var sb = new StringBuilder(); foreach (var row in group) { var line = LeanData.GenerateLine(row, _securityType, _resolution); sb.AppendLine(line); } zip.AddEntry(zipEntryName, sb.ToString()); break; } } } if (zip.Count > 0) { zip.Save(); } } date = date.AddDays(1); } }
public void ValidForexDataLinks(string ticker, string market, DateTime date, Resolution resolution, TickType tickType) { var path = LeanData.GenerateRelativeZipFilePath( new Symbol(SecurityIdentifier.GenerateForex(ticker, market), ticker), date, resolution, tickType); var dataLink = ApiClient.ReadDataLink(path, TestOrganization); Assert.IsTrue(dataLink.Success); Assert.IsFalse(dataLink.Url.IsNullOrEmpty()); }
public bool DownloadHourData(Symbol symbol, Resolution resolution, DateTime fromDate) { string r = "S5"; string filename = ""; string s = symbol.ID.Symbol.Insert(symbol.ID.Symbol.Length - 3, "_"); DateTime toDate = DateTime.UtcNow.RoundDown(TimeSpan.FromMinutes(30)); string url = null; r = "H1"; filename = string.Format("{0}.csv", symbol.ID.Symbol.ToLower()); DateTime _d = fromDate.ToUniversalTime(); string csv = ""; while (_d < toDate) { DateTime _end = _d.AddHours(5000) > toDate ? toDate : _d.AddHours(5000); url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(_end), r); RestClient client = new RestClient(); var auth = "Bearer " + _token; client.Timeout = 120000; client.BaseUrl = new Uri(url); Log.Trace(string.Format("Downloading {0}", url)); var request = new RestRequest(); request.AddHeader("content-type", "application/json"); request.AddHeader("Authorization", auth); IRestResponse response = client.Execute(request); string json = response.Content; if (response.ErrorException == null) { var curcsv = resolution == Resolution.Daily || resolution == Resolution.Hour ? JSonToCSV(json) : JSonToCSV(fromDate, json); if (curcsv != null && curcsv.Length > 0) { csv += curcsv; } } _d = _end; } if (csv.Length > 0) { // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, fromDate, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); CreateZip(path, filename, csv); return(true); } return(false); }
public void DecomposesAccordingToLeanDataFileGeneration(Arguments args) { var sourceString = LeanData.GenerateRelativeZipFilePath(args.Symbol, args.Date, args.Resolution, args.TickType); var decomposed = LeanDataPathComponents.Parse(sourceString); var expectedFileName = LeanData.GenerateZipFileName(args.Symbol, args.Date, args.Resolution, args.TickType); Assert.AreEqual(args.Symbol, decomposed.Symbol); Assert.AreEqual(args.Date, decomposed.Date); Assert.AreEqual(expectedFileName, decomposed.Filename); Assert.AreEqual(args.Symbol.ID.SecurityType, decomposed.SecurityType); Assert.AreEqual(args.Symbol.ID.Market, decomposed.Market); Assert.AreEqual(args.TickType, decomposed.TickType); Assert.AreEqual(args.Market, decomposed.Market); }
/// <summary> /// Method to download and save the data purchased through QuantConnect /// </summary> /// <param name="symbol">Symbol of security of which data will be requested.</param> /// <param name="resolution">Resolution of data requested.</param> /// <param name="date">Date of the data requested.</param> /// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns> public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) { // Get a link to the data var link = ReadDataLink(symbol, resolution, date); // Make sure the link was successfully retrieved if (!link.Success) { return(false); } // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); // Download and save the data var uri = new Uri(link.DataLink); var client = new RestClient(uri.Scheme + "://" + uri.Host); var request = new RestRequest(uri.PathAndQuery, Method.GET); // MAke a request for the data at the link var response = client.Execute(request); // If the response is JSON it doesn't contain any data, try and extract the message and write it if (response.ContentType.ToLowerInvariant() == "application/json") { try { var contentObj = JObject.Parse(response.Content); var message = contentObj["message"].Value <string>(); Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}"); } catch { Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}. Api response could not be parsed."); } return(false); } // Any other case save the content to given path response.RawBytes.SaveAs(path); return(true); }
/// <summary> /// Creates the <see cref="TextWriter"/> that writes data to csv files /// </summary> private Writer CreateTextWriter(BaseData data) { var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType); var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType) .Replace(".zip", string.Empty); var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry); var directory = new FileInfo(path).Directory.FullName; if (!Directory.Exists(directory)) { // lock before checking again lock (DirectoryCreateSync) if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } } return(new Writer(path, new StreamWriter(path))); }
public void GenerateRelativeZipFilePath(LeanDataTestParameters parameters) { var relativePath = LeanData.GenerateRelativeZipFilePath(parameters.Symbol, parameters.Date, parameters.Resolution, parameters.TickType); Assert.AreEqual(parameters.ExpectedRelativeZipFilePath, relativePath); }
private void SaveDailyOrHour( List <Symbol> symbols, Symbol canonicalSymbol, IReadOnlyDictionary <Symbol, List <BaseData> > historyBySymbol) { var zipFileName = Path.Combine( _dataDirectory, LeanData.GenerateRelativeZipFilePath(canonicalSymbol, DateTime.MinValue, _resolution, _tickType)); var folder = Path.GetDirectoryName(zipFileName); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } using (var zip = new ZipFile(zipFileName)) { foreach (var symbol in symbols) { // Load new data rows into a SortedDictionary for easy merge/update var newRows = new SortedDictionary <DateTime, string>(historyBySymbol[symbol] .ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution))); var rows = new SortedDictionary <DateTime, string>(); var zipEntryName = LeanData.GenerateZipEntryName(symbol, DateTime.MinValue, _resolution, _tickType); if (zip.ContainsEntry(zipEntryName)) { // If file exists, we load existing data and perform merge using (var stream = new MemoryStream()) { zip[zipEntryName].Extract(stream); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { string line; while ((line = reader.ReadLine()) != null) { var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter); rows[time] = line; } } } foreach (var kvp in newRows) { rows[kvp.Key] = kvp.Value; } } else { // No existing file, just use the new data rows = newRows; } // Loop through the SortedDictionary and write to zip entry var sb = new StringBuilder(); foreach (var kvp in rows) { // Build the line and append it to the file sb.AppendLine(kvp.Value); } // Write the zip entry if (sb.Length > 0) { if (zip.ContainsEntry(zipEntryName)) { zip.RemoveEntry(zipEntryName); } zip.AddEntry(zipEntryName, sb.ToString()); } } if (zip.Count > 0) { zip.Save(); } } }
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute) { var timeSpans = new Dictionary <Resolution, TimeSpan>() { { Resolution.Daily, TimeSpan.FromHours(24) }, { Resolution.Hour, TimeSpan.FromHours(1) }, }; var timeSpan = timeSpans[outputResolution]; var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >() { { TickType.Quote, () => new QuoteBarConsolidator(timeSpan) }, { TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) }, { TickType.Trade, () => new TradeBarConsolidator(timeSpan) } }; var consolidators = new Dictionary <string, IDataConsolidator>(); var configs = new Dictionary <string, SubscriptionDataConfig>(); var outputFiles = new Dictionary <string, StringBuilder>(); var futures = new Dictionary <string, Symbol>(); var date = _fromDate; while (date <= _toDate) { var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution); foreach (var future in futureChain) { if (!futures.ContainsKey(future.Value)) { futures[future.Value] = future; var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType), future, inputResolution, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType); configs[future.Value] = config; consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke(); var sb = new StringBuilder(); outputFiles[future.Value] = sb; consolidators[future.Value].DataConsolidated += (sender, bar) => { sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine); }; } var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory); var consolidator = consolidators[future.Value]; foreach (var bar in leanDataReader.Parse()) { consolidator.Update(bar); } } date = date.AddDays(1); } //write all results foreach (var consolidator in consolidators.Values) { consolidator.Scan(date); } var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType); var zipPath = Path.Combine(_dataDirectory, zip); var fi = new FileInfo(zipPath); if (!fi.Directory.Exists) { fi.Directory.Create(); } foreach (var future in futures.Values) { var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType); var sb = outputFiles[future.Value]; //Uncomment to write zip files //QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString()); Assert.IsTrue(sb.Length > 0); } }
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date) { var simpleDate = date.Date; if (resolution == Resolution.Hour) { return(DownloadHourData(symbol, resolution, simpleDate)); } string r = "S5"; string filename = ""; string s = symbol.ID.Symbol.Insert(symbol.ID.Symbol.Length - 3, "_"); DateTime todate = simpleDate.AddDays(1d).Date; string url = null; switch (resolution) { case Resolution.Minute: r = "M1"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_minute_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol }); url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r); break; case Resolution.Second: r = "S5"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_second_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol }); url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r); break; case Resolution.Tick: r = "S5"; filename = string.Format("{0}{1:D2}{2:D2}_{3}_tick_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol }); url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r); break; case Resolution.Hour: { r = "H1"; filename = string.Format("{0}.csv", symbol.ID.Symbol); DateTime _d = new DateTime(2004, 1, 1).ToUniversalTime(); url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(DateTime.UtcNow), r); break; } case Resolution.Daily: { r = "D"; filename = string.Format("{0}.csv", symbol.ID.Symbol); DateTime _d = DateTime.UtcNow.AddDays(-5000).ToUniversalTime(); url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(DateTime.UtcNow), r); break; } } RestClient client = new RestClient(); var auth = "Bearer " + _token; client.Timeout = 120000; client.BaseUrl = new Uri(url); Log.Trace(string.Format("Downloading {0}", url)); var request = new RestRequest(); request.AddHeader("content-type", "application/json"); request.AddHeader("Authorization", auth); IRestResponse response = client.Execute(request); string json = response.Content; if (response.ErrorException == null) { // Save csv in same folder heirarchy as Lean var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, simpleDate, resolution)); // Make sure the directory exist before writing (new FileInfo(path)).Directory.Create(); var csv = resolution == Resolution.Daily || resolution == Resolution.Hour ? JSonToCSV(json) : JSonToCSV(simpleDate, json); if (csv == null || csv.Length < 1) { return(false); } CreateZip(path, filename, csv, json); return(true); } Log.Trace(string.Format("Error downloading {0}", response.ErrorException)); return(false); }