/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request) { if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", $"{request.Resolution} resolution is not supported, no history returned")); yield break; } if (request.TickType != TickType.Trade) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", $"{request.TickType} tick type not supported, no history returned")); yield break; } var period = request.Resolution.ToTimeSpan(); foreach (var kline in ApiClient.GetHistory(request)) { yield return(new TradeBar() { Time = Time.UnixMillisecondTimeStampToDateTime(kline.OpenTime), Symbol = request.Symbol, Low = kline.Low, High = kline.High, Open = kline.Open, Close = kline.Close, Volume = kline.Volume, Value = kline.Close, DataType = MarketDataType.TradeBar, Period = period }); } }
/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public IEnumerable <Messages.Kline> GetHistory(Data.HistoryRequest request) { var resolution = ConvertResolution(request.Resolution); var resolutionInMs = (long)request.Resolution.ToTimeSpan().TotalMilliseconds; var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); var startMs = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000; var endMs = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000; var endpoint = $"/api/v3/klines?symbol={symbol}&interval={resolution}&limit=1000"; while (endMs - startMs >= resolutionInMs) { var timeframe = $"&startTime={startMs}&endTime={endMs}"; var restRequest = new RestRequest(endpoint + timeframe, Method.GET); var response = ExecuteRestRequest(restRequest); if (response.StatusCode != HttpStatusCode.OK) { throw new Exception($"BinanceBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}"); } var klines = JsonConvert.DeserializeObject <object[][]>(response.Content) .Select(entries => new Messages.Kline(entries)) .ToList(); startMs = klines.Last().OpenTime + resolutionInMs; foreach (var kline in klines) { yield return(kline); } } }
/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request) { if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", $"{request.Resolution} resolution not supported, no history returned")); yield break; } string resolution = ConvertResolution(request.Resolution); long resolutionInMS = (long)request.Resolution.ToTimeSpan().TotalMilliseconds; string symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); long startMTS = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000; long endMTS = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000; string endpoint = $"v2/candles/trade:{resolution}:t{symbol}/hist?limit=1000&sort=1"; while ((endMTS - startMTS) > resolutionInMS) { var timeframe = $"&start={startMTS}&end={endMTS}"; var restRequest = new RestRequest(endpoint + timeframe, Method.GET); var response = ExecuteRestRequest(restRequest); if (response.StatusCode != HttpStatusCode.OK) { throw new Exception($"BitfinexBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}"); } var candles = JsonConvert.DeserializeObject <object[][]>(response.Content) .Select(entries => new Messages.Candle(entries)) .ToList(); startMTS = candles.Last().Timestamp + resolutionInMS; var period = request.Resolution.ToTimeSpan(); foreach (var candle in candles) { yield return(new TradeBar() { Time = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp), Symbol = request.Symbol, Low = candle.Low, High = candle.High, Open = candle.Open, Close = candle.Close, Volume = candle.Volume, Value = candle.Close, DataType = MarketDataType.TradeBar, Period = period, EndTime = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp + (long)period.TotalMilliseconds) }); } } }
/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public IEnumerable <Messages.Kline> GetHistory(Data.HistoryRequest request) { var resolution = ConvertResolution(request.Resolution); var resolutionInMs = (long)request.Resolution.ToTimeSpan().TotalMilliseconds; var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); var startMs = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000; var endMs = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000; // we always use the real endpoint for history requests var endpoint = $"https://api.binance.com/api/v3/klines?symbol={symbol}&interval={resolution}&limit=1000"; while (endMs - startMs >= resolutionInMs) { var timeframe = $"&startTime={startMs}&endTime={endMs}"; var restRequest = new RestRequest(endpoint + timeframe, Method.GET); var response = ExecuteRestRequest(restRequest); if (response.StatusCode != HttpStatusCode.OK) { throw new Exception($"BinanceBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}"); } var klines = JsonConvert.DeserializeObject <object[][]>(response.Content) .Select(entries => new Messages.Kline(entries)) .ToList(); if (klines.Count > 0) { var lastValue = klines[klines.Count - 1]; if (Log.DebuggingEnabled) { var windowStartTime = Time.UnixMillisecondTimeStampToDateTime(klines[0].OpenTime); var windowEndTime = Time.UnixMillisecondTimeStampToDateTime(lastValue.OpenTime + resolutionInMs); Log.Debug($"BinanceRestApiClient.GetHistory(): Received [{symbol}] data for timeperiod from {windowStartTime.ToStringInvariant()} to {windowEndTime.ToStringInvariant()}.."); } startMs = lastValue.OpenTime + resolutionInMs; foreach (var kline in klines) { yield return(kline); } } else { // if there is no data just break break; } } }
/// <summary> /// History based warmup enumerator /// </summary> private IEnumerator <BaseData> GetHistoryWarmupEnumerator(SubscriptionRequest warmup) { IEnumerator <BaseData> result = null; try { if (warmup.IsUniverseSubscription) { result = CreateUniverseEnumerator(warmup, createUnderlyingEnumerator: GetHistoryWarmupEnumerator); } else { var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, warmup.StartTimeUtc, warmup.EndTimeUtc); result = _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).Select(slice => { try { var data = slice.Get(historyRequest.DataType); return((BaseData)data[warmup.Configuration.Symbol]); } catch (Exception e) { Log.Error(e, $"History warmup: {warmup.Configuration}"); } return(null); }).GetEnumerator(); } return(new FilterEnumerator <BaseData>(result, // don't let future data past, nor fill forward, that will be handled after merging with the file based enumerator data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward)); } catch { // some history providers could throw if they do not support a type } return(result); }
/// <summary> /// Gets the history for the requested security /// </summary> /// <param name="request">The historical data request</param> /// <returns>An enumerable of bars covering the span specified in the request</returns> public override IEnumerable <BaseData> GetHistory(Data.HistoryRequest request) { if (request.Symbol.SecurityType != SecurityType.Crypto) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType", $"{request.Symbol.SecurityType} security type not supported, no history returned")); yield break; } if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", $"{request.Resolution} resolution not supported, no history returned")); yield break; } if (request.StartTimeUtc >= request.EndTimeUtc) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange", "The history request start date must precede the end date, no history returned")); yield break; } var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); var resultionTimeSpan = request.Resolution.ToTimeSpan(); var resolutionString = ConvertResolution(request.Resolution); var resolutionTotalMilliseconds = (long)request.Resolution.ToTimeSpan().TotalMilliseconds; var endpoint = $"{ApiVersion}/candles/trade:{resolutionString}:{symbol}/hist?limit=1000&sort=1"; // Bitfinex API only allows to support trade bar history requests. // The start and end dates are expected to match exactly with the beginning of the first bar and ending of the last. // So we need to round up dates accordingly. var startTimeStamp = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc.RoundDown(resultionTimeSpan)) * 1000; var endTimeStamp = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc.RoundDown(resultionTimeSpan)) * 1000; do { var timeframe = $"&start={startTimeStamp}&end={endTimeStamp}"; var restRequest = new RestRequest(endpoint + timeframe, Method.GET); var response = ExecuteRestRequest(restRequest); if (response.StatusCode != HttpStatusCode.OK) { throw new Exception( $"BitfinexBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, " + $"Content: {response.Content}, ErrorMessage: {response.ErrorMessage}"); } // Drop the last bar provided by the exchange as its open time is a history request's end time var candles = JsonConvert.DeserializeObject <object[][]>(response.Content) .Select(entries => new Candle(entries)) .Where(candle => candle.Timestamp != endTimeStamp) .ToList(); // Bitfinex exchange may return us an empty result - if we request data for a small time interval // during which no trades occurred - so it's rational to ensure 'candles' list is not empty before // we proceed to avoid an exception to be thrown if (candles.Any()) { startTimeStamp = candles.Last().Timestamp + resolutionTotalMilliseconds; } else { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "NoHistoricalData", $"Exchange returned no data for {symbol} on history request " + $"from {request.StartTimeUtc:s} to {request.EndTimeUtc:s}")); yield break; } foreach (var candle in candles) { yield return(new TradeBar { Time = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp), Symbol = request.Symbol, Low = candle.Low, High = candle.High, Open = candle.Open, Close = candle.Close, Volume = candle.Volume, Value = candle.Close, DataType = MarketDataType.TradeBar, Period = resultionTimeSpan, EndTime = Time.UnixMillisecondTimeStampToDateTime(candle.Timestamp + (long)resultionTimeSpan.TotalMilliseconds) }); } } while (startTimeStamp < endTimeStamp); }
/// <summary> /// Populate request data /// </summary> private IEnumerable <Slice> ProcessHistoryRequests(Data.HistoryRequest request) { var ticker = request.Symbol.ID.Symbol; var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork); var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork); if (request.Resolution == Resolution.Minute && start <= DateTime.Today.AddDays(-30)) { Log.Error("IEXDataQueueHandler.GetHistory(): History calls with minute resolution for IEX available only for trailing 30 calendar days."); yield break; } if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) { Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support daily & minute resolution."); yield break; } if (start <= DateTime.Today.AddYears(-5)) { Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support a maximum of 5 years history."); yield break; } Log.Trace("IEXDataQueueHandler.ProcessHistoryRequests(): Submitting request: " + Invariant($"{request.Symbol.SecurityType}-{ticker}: {request.Resolution} {start}->{end}") ); var span = end.Date - start.Date; var suffixes = new List <string>(); if (span.Days < 30 && request.Resolution == Resolution.Minute) { var begin = start; while (begin < end) { suffixes.Add("date/" + begin.ToStringInvariant("yyyyMMdd")); begin = begin.AddDays(1); } } else if (span.Days < 30) { suffixes.Add("1m"); } else if (span.Days < 3 * 30) { suffixes.Add("3m"); } else if (span.Days < 6 * 30) { suffixes.Add("6m"); } else if (span.Days < 12 * 30) { suffixes.Add("1y"); } else if (span.Days < 24 * 30) { suffixes.Add("2y"); } else { suffixes.Add("5y"); } // Download and parse data using (var client = new System.Net.WebClient()) { foreach (var suffix in suffixes) { var response = client.DownloadString("https://cloud.iexapis.com/v1/stock/" + ticker + "/chart/" + suffix + "?token=" + _apiKey); var parsedResponse = JArray.Parse(response); foreach (var item in parsedResponse.Children()) { DateTime date; if (item["minute"] != null) { date = DateTime.ParseExact(item["date"].Value <string>(), "yyyy-MM-dd", CultureInfo.InvariantCulture); var mins = TimeSpan.ParseExact(item["minute"].Value <string>(), "hh\\:mm", CultureInfo.InvariantCulture); date += mins; } else { date = Parse.DateTime(item["date"].Value <string>()); } if (date.Date < start.Date || date.Date > end.Date) { continue; } Interlocked.Increment(ref _dataPointCount); if (item["open"].Type == JTokenType.Null) { continue; } var open = item["open"].Value <decimal>(); var high = item["high"].Value <decimal>(); var low = item["low"].Value <decimal>(); var close = item["close"].Value <decimal>(); var volume = item["volume"].Value <int>(); var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume); yield return(new Slice(tradeBar.EndTime, new[] { tradeBar })); } } } }
/// <summary> /// Populate request data /// </summary> private IEnumerable <BaseData> ProcessHistoryRequests(Data.HistoryRequest request) { var ticker = request.Symbol.ID.Symbol; var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork); var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork); if (request.Resolution == Resolution.Minute && start <= DateTime.Today.AddDays(-30)) { Log.Error("IEXDataQueueHandler.GetHistory(): History calls with minute resolution for IEX available only for trailing 30 calendar days."); yield break; } if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) { Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support daily & minute resolution."); yield break; } Log.Trace("IEXDataQueueHandler.ProcessHistoryRequests(): Submitting request: " + Invariant($"{request.Symbol.SecurityType}-{ticker}: {request.Resolution} {start}->{end}") + ". Please wait.."); const string baseUrl = "https://cloud.iexapis.com/stable/stock"; var now = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); var span = now - start; var urls = new List <string>(); switch (request.Resolution) { case Resolution.Minute: { var begin = start; while (begin < end) { var url = $"{baseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}"; urls.Add(url); begin = begin.AddDays(1); } break; } case Resolution.Daily: { string suffix; if (span.Days < 30) { suffix = "1m"; } else if (span.Days < 3 * 30) { suffix = "3m"; } else if (span.Days < 6 * 30) { suffix = "6m"; } else if (span.Days < 12 * 30) { suffix = "1y"; } else if (span.Days < 24 * 30) { suffix = "2y"; } else if (span.Days < 60 * 30) { suffix = "5y"; } else { suffix = "max"; // max is 15 years } var url = $"{baseUrl}/{ticker}/chart/{suffix}?token={_apiKey}"; urls.Add(url); break; } } // Download and parse data var requests = new List <Task <string> >(); urls.DoForEach(url => { requests.Add(Task.Run(async() => { using (var client = new WebClient()) { return(await client.DownloadStringTaskAsync(new Uri(url)).ConfigureAwait(false)); } })); }); var responses = Task.WhenAll(requests).Result; foreach (var response in responses) { var parsedResponse = JArray.Parse(response); // Parse foreach (var item in parsedResponse.Children()) { DateTime date; TimeSpan period; if (item["minute"] != null) { date = DateTime.ParseExact(item["date"].Value <string>(), "yyyy-MM-dd", CultureInfo.InvariantCulture); var minutes = TimeSpan.ParseExact(item["minute"].Value <string>(), "hh\\:mm", CultureInfo.InvariantCulture); date += minutes; period = TimeSpan.FromMinutes(1); } else { date = Parse.DateTime(item["date"].Value <string>()); period = TimeSpan.FromDays(1); } if (date < start || date > end) { continue; } Interlocked.Increment(ref _dataPointCount); if (item["open"].Type == JTokenType.Null) { continue; } decimal open, high, low, close, volume; if (request.Resolution == Resolution.Daily && request.DataNormalizationMode == DataNormalizationMode.Raw) { open = item["uOpen"].Value <decimal>(); high = item["uHigh"].Value <decimal>(); low = item["uLow"].Value <decimal>(); close = item["uClose"].Value <decimal>(); volume = item["uVolume"].Value <int>(); } else { open = item["open"].Value <decimal>(); high = item["high"].Value <decimal>(); low = item["low"].Value <decimal>(); close = item["close"].Value <decimal>(); volume = item["volume"].Value <int>(); } var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume, period); yield return(tradeBar); } } }