/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Quote)
{
yield break;
}
if (!_symbolMapper.IsKnownLeanSymbol(symbol))
{
throw new ArgumentException("Invalid symbol requested: " + symbol.Value);
}
if (symbol.ID.SecurityType != SecurityType.Forex && symbol.ID.SecurityType != SecurityType.Cfd)
{
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal to the start date.");
}
// set the starting date
DateTime date = startUtc;
// loop until last date
while (date <= endUtc)
{
// request all ticks for a specific date
var ticks = DownloadTicks(symbol, date);
switch (resolution)
{
case Resolution.Tick:
foreach (var tick in ticks)
{
yield return(new Tick(tick.Time, symbol, tick.BidPrice, tick.AskPrice));
}
break;
case Resolution.Second:
case Resolution.Minute:
case Resolution.Hour:
case Resolution.Daily:
foreach (var bar in LeanData.AggregateTicks(ticks, symbol, resolution.ToTimeSpan()))
{
yield return(bar);
}
break;
}
date = date.AddDays(1);
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
return(Enumerable.Empty <BaseData>());
}
if (resolution != Resolution.Daily)
{
throw new ArgumentException("The YahooDataDownloader can only download daily data.");
}
if (symbol.ID.SecurityType != SecurityType.Equity)
{
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
return(GetEnumerator(symbol, startUtc, endUtc));
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Quote)
{
yield break;
}
if (resolution == Resolution.Tick)
{
throw new NotSupportedException("Resolution not available: " + resolution);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var symbols = new List <Symbol> {
symbol
};
if (symbol.IsCanonical())
{
symbols = GetChainSymbols(symbol, true).ToList();
}
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
foreach (var targetSymbol in symbols)
{
var historyRequest = new HistoryRequest(startUtc,
endUtc,
typeof(QuoteBar),
targetSymbol,
resolution,
exchangeHours: exchangeHours,
dataTimeZone: dataTimeZone,
resolution,
includeExtendedMarketHours: true,
false,
DataNormalizationMode.Adjusted,
TickType.Quote);
foreach (var baseData in _brokerage.GetHistory(historyRequest))
{
yield return(baseData);
}
}
}
/// <summary>
/// Get historical data enumerable for Bitfinex from Quandl
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
yield break;
}
if (resolution != Resolution.Daily)
{
throw new ArgumentException("Only daily data is currently supported.");
}
const string collapse = "daily";
var url = "https://www.quandl.com/api/v3/datasets/BCHARTS/BITFINEXUSD.csv?order=asc&collapse=" + collapse + "&api_key=" + _apiKey + "&start_date="
+ startUtc.ToStringInvariant("yyyy-MM-dd");
using (var cl = new WebClient())
{
var data = cl.DownloadString(url);
// skip the header line
foreach (var item in data.Split('\n').Skip(1))
{
var line = item.Split(',');
if (line.Length != 8)
{
continue;
}
var bar = new TradeBar
{
Time = Parse.DateTime(line[0]),
Open = Parse.Decimal(line[1]),
High = Parse.Decimal(line[2]),
Low = Parse.Decimal(line[3]),
Close = Parse.Decimal(line[4]),
Value = Parse.Decimal(line[7]),
Volume = (long)Parse.Decimal(line[5]),
Symbol = symbol,
DataType = MarketDataType.TradeBar,
Period = Time.OneDay
};
yield return(bar);
}
}
}
/// <summary>
/// Determines if it should downloads new data and retrieves data from disc
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public override Stream Fetch(string key)
{
return(DownloadOnce(key, s =>
{
if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType))
{
var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
DateTime startTimeUtc;
DateTime endTimeUtc;
if (resolution < Resolution.Hour)
{
// we can get the date from the path
startTimeUtc = date.ConvertToUtc(dataTimeZone);
endTimeUtc = date.Add(resolution.ToTimeSpan()).ConvertToUtc(dataTimeZone);
}
else
{
try
{
startTimeUtc = symbol.ID.Date;
}
catch (InvalidOperationException)
{
startTimeUtc = Time.BeginningOfTime;
}
endTimeUtc = DateTime.UtcNow;
}
// Save the data
var writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType);
try
{
var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
var data = _dataDownloader.Get(getParams)
.Where(baseData => symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
// for canonical symbols, downloader will return data for all of the chain
.GroupBy(baseData => baseData.Symbol);
foreach (var dataPerSymbol in data)
{
writer.Write(dataPerSymbol);
}
}
catch (Exception e)
{
Log.Error(e);
}
}
}));
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Quote)
{
yield break;
}
if (resolution != Resolution.Tick)
{
throw new ArgumentException("Only tick data is currently supported.");
}
var hour = 1;
var counter = startUtc;
while (counter <= endUtc)
{
while (hour < 24)
{
using (var cl = new WebClient())
{
var request = $"http://cryptoiq.io/api/marketdata/ticker/{_exchange}/{symbol.Value}/{counter.ToStringInvariant("yyyy-MM-dd")}/{hour}";
var data = cl.DownloadString(request);
var mbtc = JsonConvert.DeserializeObject <List <CryptoiqBitcoin> >(data);
foreach (var item in mbtc.OrderBy(x => x.Time))
{
yield return(new Tick
{
Time = item.Time,
Symbol = symbol,
Value = item.Last,
AskPrice = item.Ask,
BidPrice = item.Bid,
TickType = TickType.Quote
});
}
hour++;
}
}
counter = counter.AddDays(1);
hour = 0;
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
return(Enumerable.Empty <BaseData>());
}
if (resolution == Resolution.Tick || resolution == Resolution.Second)
{
throw new ArgumentException($"Resolution not available: {resolution}");
}
if (!_symbolMapper.IsKnownLeanSymbol(symbol))
{
throw new ArgumentException($"The ticker {symbol.Value} is not available.");
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var historyRequest = new HistoryRequest(
startUtc,
endUtc,
typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.EasternStandard),
DateTimeZone.Utc,
resolution,
false,
false,
DataNormalizationMode.Adjusted,
TickType.Trade);
var data = _brokerage.GetHistory(historyRequest);
return(data);
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
return(Enumerable.Empty <BaseData>());
}
// get symbol mapper for GDAX
var mapper = new SymbolPropertiesDatabaseSymbolMapper(Market.GDAX);
var brokerageTicker = mapper.GetBrokerageSymbol(symbol);
var returnData = new List <BaseData>();
var granularity = resolution.ToTimeSpan().TotalSeconds;
DateTime windowStartTime = startUtc;
DateTime windowEndTime = startUtc;
do
{
windowStartTime = windowEndTime;
windowEndTime = windowStartTime.AddSeconds(MaxDatapointsPerRequest * granularity);
windowEndTime = windowEndTime > endUtc ? endUtc : windowEndTime;
Log.Trace($"Getting data for timeperiod from {windowStartTime.ToStringInvariant()} to {windowEndTime.ToStringInvariant()}..");
var requestURL = $"http://api.pro.coinbase.com/products/{brokerageTicker}/candles" +
$"?start={windowStartTime.ToStringInvariant()}" +
$"&end={windowEndTime.ToStringInvariant()}" +
$"&granularity={granularity.ToStringInvariant()}";
var request = (HttpWebRequest)WebRequest.Create(requestURL);
request.UserAgent = ".NET Framework Test Client";
string data = GetWithRetry(request);
returnData.AddRange(ParseCandleData(symbol, granularity, data));
}while (windowStartTime != windowEndTime);
return(returnData);
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
yield break;
}
if (!(resolution == Resolution.Daily || resolution == Resolution.Minute))
{
throw new NotSupportedException("Resolution not available: " + resolution);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var historyRequests = new[] {
new HistoryRequest(startUtc,
endUtc,
typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
resolution,
true,
false,
DataNormalizationMode.Raw,
TickType.Trade)
};
foreach (var slice in _handler.GetHistory(historyRequests, TimeZones.EasternStandard))
{
yield return(slice[symbol]);
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (symbol.SecurityType != SecurityType.Equity &&
symbol.SecurityType != SecurityType.Forex &&
symbol.SecurityType != SecurityType.Crypto)
{
throw new NotSupportedException($"Security type not supported: {symbol.SecurityType}");
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var dataType = LeanData.GetDataType(resolution, tickType);
var historyRequest =
new HistoryRequest(startUtc,
endUtc,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
resolution,
true,
false,
DataNormalizationMode.Adjusted,
tickType);
foreach (var baseData in _historyProvider.GetHistory(historyRequest))
{
yield return(baseData);
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType == TickType.OpenInterest)
{
return(Enumerable.Empty <BaseData>());
}
if (symbol.ID.SecurityType != SecurityType.Equity)
{
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var dataType = resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar);
return(_fileHistoryProvider.ProcessHistoryRequests(
new HistoryRequest(
startUtc,
endUtc,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
resolution,
true,
false,
DataNormalizationMode.Adjusted,
tickType)));
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
return(Enumerable.Empty <BaseData>());
}
var request = new RestRequest("query", DataFormat.Json);
request.AddParameter("symbol", symbol.Value);
request.AddParameter("datatype", "csv");
IEnumerable <TimeSeries> data = null;
switch (resolution)
{
case Resolution.Minute:
case Resolution.Hour:
data = GetIntradayData(request, startUtc, endUtc, resolution);
break;
case Resolution.Daily:
data = GetDailyData(request, startUtc, endUtc, symbol);
break;
default:
throw new ArgumentOutOfRangeException(nameof(resolution), $"{resolution} resolution not supported by API.");
}
var period = resolution.ToTimeSpan();
return(data.Select(d => new TradeBar(d.Time, symbol, d.Open, d.High, d.Low, d.Close, d.Volume, period)));
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
yield break;
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
if (resolution != Resolution.Tick)
{
throw new NotSupportedException("Only Tick Resolution is supported.");
}
var startUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(startUtc) * 1000000000); // Multiply by 10^9 per Kraken API
var endUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(endUtc) * 1000000000);
var url = string.Format(CultureInfo.InvariantCulture, UrlPrototype, symbol.Value, startUnixTime);
List <List <string> > data;
using (var client = new WebClient())
{
var rateGate = new RateGate(10, TimeSpan.FromMinutes(1)); // 10 calls per minute for Kraken API
rateGate.WaitToProceed();
var response = client.DownloadString(url);
dynamic result = JsonConvert.DeserializeObject <dynamic>(response);
if (result.error.Count != 0)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
if (result.result.ContainsKey(symbol.Value))
{
data = result.result[symbol.Value].ToObject <List <List <string> > >();
}
else
{
throw new NotSupportedException("Asset pair was not found in the response. Make sure you use the correct model (XBTUSD -> XXBTZUSD).");
}
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Parse.Double(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Parse.Decimal(i[0]);
var volume = Parse.Decimal(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
var last = Convert.ToInt64(result.result.last);
while (last < endUnixTime)
{
url = string.Format(UrlPrototype, symbol.Value, last);
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
var errorCount = 0;
while (result.error.Count != 0 && errorCount < 10)
{
errorCount++;
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
}
if (result.error.Count != 0 && errorCount >= 10)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
data = result.result[symbol.Value].ToObject <List <List <string> > >();
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Parse.Double(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Parse.Decimal(i[0]);
var volume = Parse.Decimal(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
last = Convert.ToInt64(result.result.last);
}
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Quote)
{
yield break;
}
if (!_symbolMapper.IsKnownLeanSymbol(symbol))
{
throw new ArgumentException("Invalid symbol requested: " + symbol.Value);
}
if (resolution == Resolution.Tick)
{
throw new NotSupportedException("Resolution not available: " + resolution);
}
if (symbol.ID.SecurityType != SecurityType.Forex && symbol.ID.SecurityType != SecurityType.Cfd)
{
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var barsTotalInPeriod = new List <QuoteBar>();
var barsToSave = new List <QuoteBar>();
// set the starting date/time
var date = startUtc;
var startDateTime = date;
// loop until last date
while (startDateTime <= endUtc.AddDays(1))
{
// request blocks of 5-second bars with a starting date/time
var bars = _brokerage.DownloadQuoteBars(symbol, startDateTime, endUtc.AddDays(1), Resolution.Second, DateTimeZone.Utc).ToList();
if (bars.Count == 0)
{
break;
}
var groupedBars = GroupBarsByDate(bars);
if (groupedBars.Count > 1)
{
// we received more than one day, so we save the completed days and continue
while (groupedBars.Count > 1)
{
var currentDate = groupedBars.Keys.First();
if (currentDate > endUtc)
{
break;
}
barsToSave.AddRange(groupedBars[currentDate]);
barsTotalInPeriod.AddRange(barsToSave);
barsToSave.Clear();
// remove the completed date
groupedBars.Remove(currentDate);
}
// update the current date
date = groupedBars.Keys.First();
if (date <= endUtc)
{
barsToSave.AddRange(groupedBars[date]);
}
}
else
{
var currentDate = groupedBars.Keys.First();
if (currentDate > endUtc)
{
break;
}
// update the current date
date = currentDate;
barsToSave.AddRange(groupedBars[date]);
}
// calculate the next request datetime (next 5-sec bar time)
startDateTime = bars[bars.Count - 1].Time.AddSeconds(5);
}
if (barsToSave.Count > 0)
{
barsTotalInPeriod.AddRange(barsToSave);
}
switch (resolution)
{
case Resolution.Second:
case Resolution.Minute:
case Resolution.Hour:
case Resolution.Daily:
foreach (var bar in LeanData.AggregateQuoteBars(barsTotalInPeriod, symbol, resolution.ToTimeSpan()))
{
yield return(bar);
}
break;
}
}