/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request)
{
if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution is not supported, no history returned"));
yield break;
}
if (request.TickType != TickType.Trade)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType",
$"{request.TickType} tick type not supported, no history returned"));
yield break;
}
var period = request.Resolution.ToTimeSpan();
foreach (var kline in ApiClient.GetHistory(request))
{
yield return(new TradeBar()
{
Time = Time.UnixMillisecondTimeStampToDateTime(kline.OpenTime),
Symbol = request.Symbol,
Low = kline.Low,
High = kline.High,
Open = kline.Open,
Close = kline.Close,
Volume = kline.Volume,
Value = kline.Close,
DataType = MarketDataType.TradeBar,
Period = period
});
}
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public IEnumerable <Messages.Kline> GetHistory(Data.HistoryRequest request)
{
var resolution = ConvertResolution(request.Resolution);
var resolutionInMs = (long)request.Resolution.ToTimeSpan().TotalMilliseconds;
var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
var startMs = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000;
var endMs = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000;
var endpoint = $"/api/v3/klines?symbol={symbol}&interval={resolution}&limit=1000";
while (endMs - startMs >= resolutionInMs)
{
var timeframe = $"&startTime={startMs}&endTime={endMs}";
var restRequest = new RestRequest(endpoint + timeframe, Method.GET);
var response = ExecuteRestRequest(restRequest);
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"BinanceBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
}
var klines = JsonConvert.DeserializeObject <object[][]>(response.Content)
.Select(entries => new Messages.Kline(entries))
.ToList();
startMs = klines.Last().OpenTime + resolutionInMs;
foreach (var kline in klines)
{
yield return(kline);
}
}
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request)
{
if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution not supported, no history returned"));
yield break;
}
string resolution = ConvertResolution(request.Resolution);
long resolutionInMS = (long)request.Resolution.ToTimeSpan().TotalMilliseconds;
string symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
long startMTS = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000;
long endMTS = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000;
string endpoint = $"v2/candles/trade:{resolution}:t{symbol}/hist?limit=1000&sort=1";
while ((endMTS - startMTS) > resolutionInMS)
{
var timeframe = $"&start={startMTS}&end={endMTS}";
var restRequest = new RestRequest(endpoint + timeframe, Method.GET);
var response = ExecuteRestRequest(restRequest);
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"BitfinexBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
}
var candles = JsonConvert.DeserializeObject <object[][]>(response.Content)
.Select(entries => new Messages.Candle(entries))
.ToList();
startMTS = candles.Last().Timestamp + resolutionInMS;
var period = request.Resolution.ToTimeSpan();
foreach (var candle in candles)
{
yield return(new TradeBar()
{
Time = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp),
Symbol = request.Symbol,
Low = candle.Low,
High = candle.High,
Open = candle.Open,
Close = candle.Close,
Volume = candle.Volume,
Value = candle.Close,
DataType = MarketDataType.TradeBar,
Period = period,
EndTime = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp + (long)period.TotalMilliseconds)
});
}
}
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public IEnumerable <Messages.Kline> GetHistory(Data.HistoryRequest request)
{
var resolution = ConvertResolution(request.Resolution);
var resolutionInMs = (long)request.Resolution.ToTimeSpan().TotalMilliseconds;
var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
var startMs = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000;
var endMs = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000;
// we always use the real endpoint for history requests
var endpoint = $"https://api.binance.com/api/v3/klines?symbol={symbol}&interval={resolution}&limit=1000";
while (endMs - startMs >= resolutionInMs)
{
var timeframe = $"&startTime={startMs}&endTime={endMs}";
var restRequest = new RestRequest(endpoint + timeframe, Method.GET);
var response = ExecuteRestRequest(restRequest);
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"BinanceBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
}
var klines = JsonConvert.DeserializeObject <object[][]>(response.Content)
.Select(entries => new Messages.Kline(entries))
.ToList();
if (klines.Count > 0)
{
var lastValue = klines[klines.Count - 1];
if (Log.DebuggingEnabled)
{
var windowStartTime = Time.UnixMillisecondTimeStampToDateTime(klines[0].OpenTime);
var windowEndTime = Time.UnixMillisecondTimeStampToDateTime(lastValue.OpenTime + resolutionInMs);
Log.Debug($"BinanceRestApiClient.GetHistory(): Received [{symbol}] data for timeperiod from {windowStartTime.ToStringInvariant()} to {windowEndTime.ToStringInvariant()}..");
}
startMs = lastValue.OpenTime + resolutionInMs;
foreach (var kline in klines)
{
yield return(kline);
}
}
else
{
// if there is no data just break
break;
}
}
}
/// <summary>
/// History based warmup enumerator
/// </summary>
private IEnumerator <BaseData> GetHistoryWarmupEnumerator(SubscriptionRequest warmup)
{
IEnumerator <BaseData> result = null;
try
{
if (warmup.IsUniverseSubscription)
{
result = CreateUniverseEnumerator(warmup, createUnderlyingEnumerator: GetHistoryWarmupEnumerator);
}
else
{
var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, warmup.StartTimeUtc, warmup.EndTimeUtc);
result = _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).Select(slice =>
{
try
{
var data = slice.Get(historyRequest.DataType);
return((BaseData)data[warmup.Configuration.Symbol]);
}
catch (Exception e)
{
Log.Error(e, $"History warmup: {warmup.Configuration}");
}
return(null);
}).GetEnumerator();
}
return(new FilterEnumerator <BaseData>(result,
// don't let future data past, nor fill forward, that will be handled after merging with the file based enumerator
data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward));
}
catch
{
// some history providers could throw if they do not support a type
}
return(result);
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request)
{
if (request.Symbol.SecurityType != SecurityType.Crypto)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType",
$"{request.Symbol.SecurityType} security type not supported, no history returned"));
yield break;
}
if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution not supported, no history returned"));
yield break;
}
if (request.StartTimeUtc >= request.EndTimeUtc)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange",
"The history request start date must precede the end date, no history returned"));
yield break;
}
var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
var resultionTimeSpan = request.Resolution.ToTimeSpan();
var resolutionString = ConvertResolution(request.Resolution);
var resolutionTotalMilliseconds = (long)request.Resolution.ToTimeSpan().TotalMilliseconds;
var endpoint = $"{ApiVersion}/candles/trade:{resolutionString}:{symbol}/hist?limit=1000&sort=1";
// Bitfinex API only allows to support trade bar history requests.
// The start and end dates are expected to match exactly with the beginning of the first bar and ending of the last.
// So we need to round up dates accordingly.
var startTimeStamp = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc.RoundDown(resultionTimeSpan)) * 1000;
var endTimeStamp = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc.RoundDown(resultionTimeSpan)) * 1000;
do
{
var timeframe = $"&start={startTimeStamp}&end={endTimeStamp}";
var restRequest = new RestRequest(endpoint + timeframe, Method.GET);
var response = ExecuteRestRequest(restRequest);
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception(
$"BitfinexBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, " +
$"Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
}
// Drop the last bar provided by the exchange as its open time is a history request's end time
var candles = JsonConvert.DeserializeObject <object[][]>(response.Content)
.Select(entries => new Candle(entries))
.Where(candle => candle.Timestamp != endTimeStamp)
.ToList();
// Bitfinex exchange may return us an empty result - if we request data for a small time interval
// during which no trades occurred - so it's rational to ensure 'candles' list is not empty before
// we proceed to avoid an exception to be thrown
if (candles.Any())
{
startTimeStamp = candles.Last().Timestamp + resolutionTotalMilliseconds;
}
else
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "NoHistoricalData",
$"Exchange returned no data for {symbol} on history request " +
$"from {request.StartTimeUtc:s} to {request.EndTimeUtc:s}"));
yield break;
}
foreach (var candle in candles)
{
yield return(new TradeBar
{
Time = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp),
Symbol = request.Symbol,
Low = candle.Low,
High = candle.High,
Open = candle.Open,
Close = candle.Close,
Volume = candle.Volume,
Value = candle.Close,
DataType = MarketDataType.TradeBar,
Period = resultionTimeSpan,
EndTime = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp + (long)resultionTimeSpan.TotalMilliseconds)
});
}
} while (startTimeStamp < endTimeStamp);
}
/// <summary>
/// Populate request data
/// </summary>
private IEnumerable <Slice> ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork);
var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork);
if (request.Resolution == Resolution.Minute && start <= DateTime.Today.AddDays(-30))
{
Log.Error("IEXDataQueueHandler.GetHistory(): History calls with minute resolution for IEX available only for trailing 30 calendar days.");
yield break;
}
if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support daily & minute resolution.");
yield break;
}
if (start <= DateTime.Today.AddYears(-5))
{
Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support a maximum of 5 years history.");
yield break;
}
Log.Trace("IEXDataQueueHandler.ProcessHistoryRequests(): Submitting request: " +
Invariant($"{request.Symbol.SecurityType}-{ticker}: {request.Resolution} {start}->{end}")
);
var span = end.Date - start.Date;
var suffixes = new List <string>();
if (span.Days < 30 && request.Resolution == Resolution.Minute)
{
var begin = start;
while (begin < end)
{
suffixes.Add("date/" + begin.ToStringInvariant("yyyyMMdd"));
begin = begin.AddDays(1);
}
}
else if (span.Days < 30)
{
suffixes.Add("1m");
}
else if (span.Days < 3 * 30)
{
suffixes.Add("3m");
}
else if (span.Days < 6 * 30)
{
suffixes.Add("6m");
}
else if (span.Days < 12 * 30)
{
suffixes.Add("1y");
}
else if (span.Days < 24 * 30)
{
suffixes.Add("2y");
}
else
{
suffixes.Add("5y");
}
// Download and parse data
using (var client = new System.Net.WebClient())
{
foreach (var suffix in suffixes)
{
var response = client.DownloadString("https://cloud.iexapis.com/v1/stock/" + ticker + "/chart/" + suffix + "?token=" + _apiKey);
var parsedResponse = JArray.Parse(response);
foreach (var item in parsedResponse.Children())
{
DateTime date;
if (item["minute"] != null)
{
date = DateTime.ParseExact(item["date"].Value <string>(), "yyyy-MM-dd", CultureInfo.InvariantCulture);
var mins = TimeSpan.ParseExact(item["minute"].Value <string>(), "hh\\:mm", CultureInfo.InvariantCulture);
date += mins;
}
else
{
date = Parse.DateTime(item["date"].Value <string>());
}
if (date.Date < start.Date || date.Date > end.Date)
{
continue;
}
Interlocked.Increment(ref _dataPointCount);
if (item["open"].Type == JTokenType.Null)
{
continue;
}
var open = item["open"].Value <decimal>();
var high = item["high"].Value <decimal>();
var low = item["low"].Value <decimal>();
var close = item["close"].Value <decimal>();
var volume = item["volume"].Value <int>();
var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume);
yield return(new Slice(tradeBar.EndTime, new[] { tradeBar }));
}
}
}
}
/// <summary>
/// Populate request data
/// </summary>
private IEnumerable <BaseData> ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork);
var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork);
if (request.Resolution == Resolution.Minute && start <= DateTime.Today.AddDays(-30))
{
Log.Error("IEXDataQueueHandler.GetHistory(): History calls with minute resolution for IEX available only for trailing 30 calendar days.");
yield break;
}
if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support daily & minute resolution.");
yield break;
}
Log.Trace("IEXDataQueueHandler.ProcessHistoryRequests(): Submitting request: " +
Invariant($"{request.Symbol.SecurityType}-{ticker}: {request.Resolution} {start}->{end}") +
". Please wait..");
const string baseUrl = "https://cloud.iexapis.com/stable/stock";
var now = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var span = now - start;
var urls = new List <string>();
switch (request.Resolution)
{
case Resolution.Minute:
{
var begin = start;
while (begin < end)
{
var url =
$"{baseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}";
urls.Add(url);
begin = begin.AddDays(1);
}
break;
}
case Resolution.Daily:
{
string suffix;
if (span.Days < 30)
{
suffix = "1m";
}
else if (span.Days < 3 * 30)
{
suffix = "3m";
}
else if (span.Days < 6 * 30)
{
suffix = "6m";
}
else if (span.Days < 12 * 30)
{
suffix = "1y";
}
else if (span.Days < 24 * 30)
{
suffix = "2y";
}
else if (span.Days < 60 * 30)
{
suffix = "5y";
}
else
{
suffix = "max"; // max is 15 years
}
var url =
$"{baseUrl}/{ticker}/chart/{suffix}?token={_apiKey}";
urls.Add(url);
break;
}
}
// Download and parse data
var requests = new List <Task <string> >();
urls.DoForEach(url =>
{
requests.Add(Task.Run(async() =>
{
using (var client = new WebClient())
{
return(await client.DownloadStringTaskAsync(new Uri(url)).ConfigureAwait(false));
}
}));
});
var responses = Task.WhenAll(requests).Result;
foreach (var response in responses)
{
var parsedResponse = JArray.Parse(response);
// Parse
foreach (var item in parsedResponse.Children())
{
DateTime date;
TimeSpan period;
if (item["minute"] != null)
{
date = DateTime.ParseExact(item["date"].Value <string>(), "yyyy-MM-dd", CultureInfo.InvariantCulture);
var minutes = TimeSpan.ParseExact(item["minute"].Value <string>(), "hh\\:mm", CultureInfo.InvariantCulture);
date += minutes;
period = TimeSpan.FromMinutes(1);
}
else
{
date = Parse.DateTime(item["date"].Value <string>());
period = TimeSpan.FromDays(1);
}
if (date < start || date > end)
{
continue;
}
Interlocked.Increment(ref _dataPointCount);
if (item["open"].Type == JTokenType.Null)
{
continue;
}
decimal open, high, low, close, volume;
if (request.Resolution == Resolution.Daily &&
request.DataNormalizationMode == DataNormalizationMode.Raw)
{
open = item["uOpen"].Value <decimal>();
high = item["uHigh"].Value <decimal>();
low = item["uLow"].Value <decimal>();
close = item["uClose"].Value <decimal>();
volume = item["uVolume"].Value <int>();
}
else
{
open = item["open"].Value <decimal>();
high = item["high"].Value <decimal>();
low = item["low"].Value <decimal>();
close = item["close"].Value <decimal>();
volume = item["volume"].Value <int>();
}
var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume, period);
yield return(tradeBar);
}
}
}