/// <summary>
/// Method to download and save the data purchased through QuantConnect
/// </summary>
/// <param name="symbol">Symbol of security of which data will be requested.</param>
/// <param name="resolution">Resolution of data requested.</param>
/// <param name="date">Date of the data requested.</param>
/// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns>
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
// Get a link to the data
var link = ReadDataLink(symbol, resolution, date);
// Make sure the link was successfully retrieved
if (!link.Success)
{
return(false);
}
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
// Download and save the data
var uri = new Uri(link.DataLink);
var client = new RestClient(uri.Scheme + "://" + uri.Host);
var request = new RestRequest(uri.PathAndQuery, Method.GET);
// If the response is not a zip then it is not data, don't write it.
var response = client.Execute(request);
if (response.ContentType != "application/zip")
{
var message = JObject.Parse(response.Content)["message"].Value <string>();
Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}");
return(false);
}
response.RawBytes.SaveAs(path);
return(true);
}
/// <summary>
/// Method to download and save the data purchased through QuantConnect
/// </summary>
/// <param name="symbol">Symbol of security of which data will be requested.</param>
/// <param name="resolution">Resolution of data requested.</param>
/// <param name="date">Date of the data requested.</param>
/// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns>
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
// Get a link to the data
var link = ReadDataLink(symbol, resolution, date);
// Make sure the link was successfully retrieved
if (!link.Success)
{
return(false);
}
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
// Download and save the data
var uri = new Uri(link.DataLink);
var client = new RestClient(uri.Scheme + "://" + uri.Host);
var request = new RestRequest(uri.PathAndQuery, Method.GET);
client.DownloadData(request).SaveAs(path);
return(true);
}
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
string r = "S5";
string filename = "";
switch (resolution)
{
case Resolution.Minute:
r = "M1";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_minute_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol });
break;
case Resolution.Second:
r = "S5";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_second_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol });
break;
case Resolution.Tick:
r = "S5";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_tick_quote.csv", new object[] { date.Year, date.Month, date.Day, symbol.ID.Symbol });
break;
case Resolution.Hour:
r = "H1";
filename = string.Format("{0}.csv", symbol.ID.Symbol);
break;
case Resolution.Daily:
r = "D";
filename = string.Format("{0}.csv", symbol.ID.Symbol);
break;
}
RestClient client = new RestClient();
var auth = "Bearer " + _token;
client.Timeout = 120000;
string s = string.Format("{0}_{1}", symbol.ID.Symbol.Substring(0, symbol.ID.Symbol.Length - 3), symbol.ID.Symbol.Substring(symbol.ID.Symbol.Length - 3));
DateTime todate = date.ToUniversalTime().AddDays(1);
string url = string.Format(_url, s, _price, ToUnixTimestamp(date.ToUniversalTime()), ToUnixTimestamp(todate), r);
client.BaseUrl = new Uri(url);
Log.Trace(string.Format("Downloading {0}", url));
var request = new RestRequest();
request.AddHeader("content-type", "application/json");
request.AddHeader("Authorization", auth);
IRestResponse response = client.Execute(request);
string json = response.Content;
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
CreateZip(path, filename, JSonToCSV(date, json), json);
return(true);
}
private void SaveMinuteOrSecondOrTick(
List <Symbol> symbols,
DateTime startTimeUtc,
DateTime endTimeUtc,
Symbol canonicalSymbol,
IReadOnlyDictionary <Symbol, List <IGrouping <DateTime, BaseData> > > historyBySymbol)
{
var date = startTimeUtc;
while (date <= endTimeUtc)
{
var zipFileName = Path.Combine(
Globals.DataFolder,
LeanData.GenerateRelativeZipFilePath(canonicalSymbol, date, _resolution, _tickType));
var folder = Path.GetDirectoryName(zipFileName);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
if (File.Exists(zipFileName))
{
File.Delete(zipFileName);
}
using (var zip = new ZipFile(zipFileName))
{
foreach (var symbol in symbols)
{
var zipEntryName = LeanData.GenerateZipEntryName(symbol, date, _resolution, _tickType);
foreach (var group in historyBySymbol[symbol])
{
if (group.Key == date)
{
var sb = new StringBuilder();
foreach (var row in group)
{
var line = LeanData.GenerateLine(row, _securityType, _resolution);
sb.AppendLine(line);
}
zip.AddEntry(zipEntryName, sb.ToString());
break;
}
}
}
if (zip.Count > 0)
{
zip.Save();
}
}
date = date.AddDays(1);
}
}
public void ValidForexDataLinks(string ticker, string market, DateTime date, Resolution resolution, TickType tickType)
{
var path = LeanData.GenerateRelativeZipFilePath(
new Symbol(SecurityIdentifier.GenerateForex(ticker, market), ticker),
date, resolution, tickType);
var dataLink = ApiClient.ReadDataLink(path, TestOrganization);
Assert.IsTrue(dataLink.Success);
Assert.IsFalse(dataLink.Url.IsNullOrEmpty());
}
public bool DownloadHourData(Symbol symbol, Resolution resolution, DateTime fromDate)
{
string r = "S5";
string filename = "";
string s = symbol.ID.Symbol.Insert(symbol.ID.Symbol.Length - 3, "_");
DateTime toDate = DateTime.UtcNow.RoundDown(TimeSpan.FromMinutes(30));
string url = null;
r = "H1";
filename = string.Format("{0}.csv", symbol.ID.Symbol.ToLower());
DateTime _d = fromDate.ToUniversalTime();
string csv = "";
while (_d < toDate)
{
DateTime _end = _d.AddHours(5000) > toDate ? toDate : _d.AddHours(5000);
url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(_end), r);
RestClient client = new RestClient();
var auth = "Bearer " + _token;
client.Timeout = 120000;
client.BaseUrl = new Uri(url);
Log.Trace(string.Format("Downloading {0}", url));
var request = new RestRequest();
request.AddHeader("content-type", "application/json");
request.AddHeader("Authorization", auth);
IRestResponse response = client.Execute(request);
string json = response.Content;
if (response.ErrorException == null)
{
var curcsv = resolution == Resolution.Daily || resolution == Resolution.Hour ? JSonToCSV(json) : JSonToCSV(fromDate, json);
if (curcsv != null && curcsv.Length > 0)
{
csv += curcsv;
}
}
_d = _end;
}
if (csv.Length > 0)
{
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, fromDate, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
CreateZip(path, filename, csv);
return(true);
}
return(false);
}
public void DecomposesAccordingToLeanDataFileGeneration(Arguments args)
{
var sourceString = LeanData.GenerateRelativeZipFilePath(args.Symbol, args.Date, args.Resolution, args.TickType);
var decomposed = LeanDataPathComponents.Parse(sourceString);
var expectedFileName = LeanData.GenerateZipFileName(args.Symbol, args.Date, args.Resolution, args.TickType);
Assert.AreEqual(args.Symbol, decomposed.Symbol);
Assert.AreEqual(args.Date, decomposed.Date);
Assert.AreEqual(expectedFileName, decomposed.Filename);
Assert.AreEqual(args.Symbol.ID.SecurityType, decomposed.SecurityType);
Assert.AreEqual(args.Symbol.ID.Market, decomposed.Market);
Assert.AreEqual(args.TickType, decomposed.TickType);
Assert.AreEqual(args.Market, decomposed.Market);
}
/// <summary>
/// Method to download and save the data purchased through QuantConnect
/// </summary>
/// <param name="symbol">Symbol of security of which data will be requested.</param>
/// <param name="resolution">Resolution of data requested.</param>
/// <param name="date">Date of the data requested.</param>
/// <returns>A <see cref="bool"/> indicating whether the data was successfully downloaded or not.</returns>
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
// Get a link to the data
var link = ReadDataLink(symbol, resolution, date);
// Make sure the link was successfully retrieved
if (!link.Success)
{
return(false);
}
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataFolder, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, date, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
// Download and save the data
var uri = new Uri(link.DataLink);
var client = new RestClient(uri.Scheme + "://" + uri.Host);
var request = new RestRequest(uri.PathAndQuery, Method.GET);
// MAke a request for the data at the link
var response = client.Execute(request);
// If the response is JSON it doesn't contain any data, try and extract the message and write it
if (response.ContentType.ToLowerInvariant() == "application/json")
{
try
{
var contentObj = JObject.Parse(response.Content);
var message = contentObj["message"].Value <string>();
Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}");
}
catch
{
Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}. Api response could not be parsed.");
}
return(false);
}
// Any other case save the content to given path
response.RawBytes.SaveAs(path);
return(true);
}
/// <summary>
/// Creates the <see cref="TextWriter"/> that writes data to csv files
/// </summary>
private Writer CreateTextWriter(BaseData data)
{
var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType);
var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType)
.Replace(".zip", string.Empty);
var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry);
var directory = new FileInfo(path).Directory.FullName;
if (!Directory.Exists(directory))
{
// lock before checking again
lock (DirectoryCreateSync) if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
}
return(new Writer(path, new StreamWriter(path)));
}
public void GenerateRelativeZipFilePath(LeanDataTestParameters parameters)
{
var relativePath = LeanData.GenerateRelativeZipFilePath(parameters.Symbol, parameters.Date, parameters.Resolution, parameters.TickType);
Assert.AreEqual(parameters.ExpectedRelativeZipFilePath, relativePath);
}
private void SaveDailyOrHour(
List <Symbol> symbols,
Symbol canonicalSymbol,
IReadOnlyDictionary <Symbol, List <BaseData> > historyBySymbol)
{
var zipFileName = Path.Combine(
_dataDirectory,
LeanData.GenerateRelativeZipFilePath(canonicalSymbol, DateTime.MinValue, _resolution, _tickType));
var folder = Path.GetDirectoryName(zipFileName);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
using (var zip = new ZipFile(zipFileName))
{
foreach (var symbol in symbols)
{
// Load new data rows into a SortedDictionary for easy merge/update
var newRows = new SortedDictionary <DateTime, string>(historyBySymbol[symbol]
.ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution)));
var rows = new SortedDictionary <DateTime, string>();
var zipEntryName = LeanData.GenerateZipEntryName(symbol, DateTime.MinValue, _resolution, _tickType);
if (zip.ContainsEntry(zipEntryName))
{
// If file exists, we load existing data and perform merge
using (var stream = new MemoryStream())
{
zip[zipEntryName].Extract(stream);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter);
rows[time] = line;
}
}
}
foreach (var kvp in newRows)
{
rows[kvp.Key] = kvp.Value;
}
}
else
{
// No existing file, just use the new data
rows = newRows;
}
// Loop through the SortedDictionary and write to zip entry
var sb = new StringBuilder();
foreach (var kvp in rows)
{
// Build the line and append it to the file
sb.AppendLine(kvp.Value);
}
// Write the zip entry
if (sb.Length > 0)
{
if (zip.ContainsEntry(zipEntryName))
{
zip.RemoveEntry(zipEntryName);
}
zip.AddEntry(zipEntryName, sb.ToString());
}
}
if (zip.Count > 0)
{
zip.Save();
}
}
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary <Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24) },
{ Resolution.Hour, TimeSpan.FromHours(1) },
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >()
{
{ TickType.Quote, () => new QuoteBarConsolidator(timeSpan) },
{ TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) },
{ TickType.Trade, () => new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary <string, IDataConsolidator>();
var configs = new Dictionary <string, SubscriptionDataConfig>();
var outputFiles = new Dictionary <string, StringBuilder>();
var futures = new Dictionary <string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var consolidator = consolidators[future.Value];
foreach (var bar in leanDataReader.Parse())
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
{
consolidator.Scan(date);
}
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}
public bool DownloadData(Symbol symbol, Resolution resolution, DateTime date)
{
var simpleDate = date.Date;
if (resolution == Resolution.Hour)
{
return(DownloadHourData(symbol, resolution, simpleDate));
}
string r = "S5";
string filename = "";
string s = symbol.ID.Symbol.Insert(symbol.ID.Symbol.Length - 3, "_");
DateTime todate = simpleDate.AddDays(1d).Date;
string url = null;
switch (resolution)
{
case Resolution.Minute:
r = "M1";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_minute_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol });
url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r);
break;
case Resolution.Second:
r = "S5";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_second_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol });
url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r);
break;
case Resolution.Tick:
r = "S5";
filename = string.Format("{0}{1:D2}{2:D2}_{3}_tick_quote.csv", new object[] { simpleDate.Year, simpleDate.Month, simpleDate.Day, symbol.ID.Symbol });
url = string.Format(_url, s, _price, ToUnixTimestamp(simpleDate), ToUnixTimestamp(todate), r);
break;
case Resolution.Hour:
{
r = "H1";
filename = string.Format("{0}.csv", symbol.ID.Symbol);
DateTime _d = new DateTime(2004, 1, 1).ToUniversalTime();
url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(DateTime.UtcNow), r);
break;
}
case Resolution.Daily:
{
r = "D";
filename = string.Format("{0}.csv", symbol.ID.Symbol);
DateTime _d = DateTime.UtcNow.AddDays(-5000).ToUniversalTime();
url = string.Format(_url, s, _price, ToUnixTimestamp(_d), ToUnixTimestamp(DateTime.UtcNow), r);
break;
}
}
RestClient client = new RestClient();
var auth = "Bearer " + _token;
client.Timeout = 120000;
client.BaseUrl = new Uri(url);
Log.Trace(string.Format("Downloading {0}", url));
var request = new RestRequest();
request.AddHeader("content-type", "application/json");
request.AddHeader("Authorization", auth);
IRestResponse response = client.Execute(request);
string json = response.Content;
if (response.ErrorException == null)
{
// Save csv in same folder heirarchy as Lean
var path = Path.Combine(_dataPath, LeanData.GenerateRelativeZipFilePath(symbol.Value, symbol.ID.SecurityType, symbol.ID.Market, simpleDate, resolution));
// Make sure the directory exist before writing
(new FileInfo(path)).Directory.Create();
var csv = resolution == Resolution.Daily || resolution == Resolution.Hour ? JSonToCSV(json) : JSonToCSV(simpleDate, json);
if (csv == null || csv.Length < 1)
{
return(false);
}
CreateZip(path, filename, csv, json);
return(true);
}
Log.Trace(string.Format("Error downloading {0}", response.ErrorException));
return(false);
}