private IEnumerable <Slice> GetHistoryHour(HistoryRequest request, DateTime start, DateTime end)
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.FifteenMinutes);
if (history == null)
{
return(Enumerable.Empty <Slice>());
}
// aggregate 15 minute bars into hourly bars
var result = history
.Select(bar => new TradeBar(bar.Time, symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, Time.OneHour))
.GroupBy(x => x.Time.RoundDown(Time.OneHour))
.Select(g => new TradeBar(
g.Key,
symbol,
g.First().Open,
g.Max(t => t.High),
g.Min(t => t.Low),
g.Last().Close,
g.Sum(t => t.Volume),
Time.OneHour))
.Select(tradeBar => new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)))
.ToList();
DataPointCount += result.Count;
return(result);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.SecurityType,
request.Symbol,
request.Resolution,
request.Market,
request.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, 1.0m);
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
reader = new FillForwardEnumerator(reader, security.Exchange, request.FillForwardResolution.Value.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
return(new Subscription(security, reader, start, end, false, false));
}
private IEnumerable <Tick> GetForexQuoteTicks(HistoryRequest request)
{
// https://api.polygon.io/v1/historic/forex/EUR/USD/2020-08-24?apiKey=
var start = request.StartTimeUtc;
var end = request.EndTimeUtc;
while (start <= end)
{
using (var client = new WebClient())
{
string baseCurrency;
string quoteCurrency;
Forex.DecomposeCurrencyPair(request.Symbol.Value, out baseCurrency, out quoteCurrency);
var offset = Convert.ToInt64(Time.DateTimeToUnixTimeStampMilliseconds(start));
var url = $"{HistoryBaseUrl}/v1/historic/forex/{baseCurrency}/{quoteCurrency}/{start.Date:yyyy-MM-dd}?apiKey={_apiKey}&offset={offset}";
var response = client.DownloadString(url);
var obj = JObject.Parse(response);
var objTicks = obj["ticks"];
if (objTicks.Type == JTokenType.Null)
{
// current date finished, move to next day
start = start.Date.AddDays(1);
continue;
}
foreach (var objTick in objTicks)
{
var row = objTick.ToObject <ForexQuoteTickResponse>();
var utcTime = Time.UnixMillisecondTimeStampToDateTime(row.Timestamp);
if (utcTime < start)
{
continue;
}
start = utcTime.AddMilliseconds(1);
if (utcTime > end)
{
yield break;
}
var time = GetTickTime(request.Symbol, utcTime);
yield return(new Tick(time, request.Symbol, row.Bid, row.Ask));
}
}
}
}
private IEnumerable <Slice> GetHistoryDaily(HistoryRequest request, DateTime start, DateTime end)
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetHistoricalData(symbol, start, end);
DataPointCount += history.Count;
return(history
.Select(bar => new TradeBar(bar.Time, symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, Time.OneDay))
.Select(tradeBar => new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz))));
}
private IEnumerable <Tick> GetEquityTradeTicks(HistoryRequest request)
{
// https://api.polygon.io/v2/ticks/stocks/trades/SPY/2020-08-24?apiKey=
var start = request.StartTimeUtc;
var end = request.EndTimeUtc;
while (start <= end)
{
using (var client = new WebClient())
{
var offset = Time.DateTimeToUnixTimeStampNanoseconds(start);
var url = $"{HistoryBaseUrl}/v2/ticks/stocks/trades/{request.Symbol.Value}/{start.Date:yyyy-MM-dd}?apiKey={_apiKey}×tamp={offset}";
var response = client.DownloadString(url);
var obj = JObject.Parse(response);
var objTicks = obj["results"];
if (objTicks.Type == JTokenType.Null || !objTicks.Any())
{
// current date finished, move to next day
start = start.Date.AddDays(1);
continue;
}
foreach (var objTick in objTicks)
{
var row = objTick.ToObject <EquityTradeTickResponse>();
var utcTime = Time.UnixNanosecondTimeStampToDateTime(row.ExchangeTimestamp);
if (utcTime < start)
{
continue;
}
start = utcTime.AddMilliseconds(1);
if (utcTime > end)
{
yield break;
}
var time = GetTickTime(request.Symbol, utcTime);
yield return(new Tick(time, request.Symbol, string.Empty, string.Empty, row.Size, row.Price));
}
}
}
}
private IEnumerable <Slice> GetHistoryMinute(HistoryRequest request, DateTime start, DateTime end)
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.OneMinute);
if (history == null)
{
return(Enumerable.Empty <Slice>());
}
DataPointCount += history.Count;
return(history
.Select(bar => new TradeBar(bar.Time, symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, Time.OneMinute))
.Select(tradeBar => new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz))));
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(HistoryRequest request)
{
// TradeBar
Symbol leanSymbol = request.Symbol;
string krakenSymbol = SymbolMapper.GetBrokerageSymbol(leanSymbol);
long startTime = ((DateTimeOffset)request.StartTimeUtc).ToUnixTimeSeconds();
long endTime = ((DateTimeOffset)request.EndTimeUtc).ToUnixTimeSeconds();
TickType tickType = request.TickType;
Resolution resolution = request.Resolution;
int interval = ResolutionToInterval(resolution);
DateTimeZone zone = request.DataTimeZone;
Type dataType = request.DataType;
while (startTime > endTime)
{
GetOHLCResult result = _restApi.GetOHLC(krakenSymbol, interval, (int)startTime);
startTime = result.Last;
Dictionary <string, List <OHLC> > dict = result.Pairs;
List <OHLC> list = dict[krakenSymbol];
foreach (OHLC candle in list)
{
if (candle.Time <= endTime)
{
yield return(new TradeBar(FromUnix(candle.Time), leanSymbol, candle.Open, candle.High, candle.Low, candle.Close, candle.Volume, TimeSpan.FromMinutes(interval)));
}
}
}
yield return(null);
}
/// <summary>
/// Creates a new history request
/// </summary>
/// <param name="subscription">The config </param>
/// <param name="startAlgoTz">History request start time in algorithm time zone</param>
/// <param name="endAlgoTz">History request end time in algorithm time zone</param>
/// <param name="exchangeHours">Security exchange hours</param>
/// <param name="resolution">The resolution to use. If null will use <see cref="SubscriptionDataConfig.Resolution"/></param>
/// <returns>The new <see cref="HistoryRequest"/></returns>
public HistoryRequest CreateHistoryRequest(SubscriptionDataConfig subscription,
DateTime startAlgoTz,
DateTime endAlgoTz,
SecurityExchangeHours exchangeHours,
Resolution?resolution)
{
resolution ??= subscription.Resolution;
var request = new HistoryRequest(subscription,
exchangeHours,
startAlgoTz.ConvertToUtc(_algorithm.TimeZone),
endAlgoTz.ConvertToUtc(_algorithm.TimeZone))
{
DataType = subscription.Type,
Resolution = resolution.Value,
FillForwardResolution = subscription.FillDataForward ? resolution : null,
TickType = subscription.TickType
};
return(request);
}
private IEnumerable <Slice> GetHistorySecond(HistoryRequest request, DateTime start, DateTime end)
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.Tick);
if (history == null)
{
return(Enumerable.Empty <Slice>());
}
// aggregate ticks into 1 second bars
var result = history
.Select(tick => new Tick
{
Time = tick.Time,
Symbol = symbol,
Value = tick.Price,
TickType = TickType.Trade,
Quantity = Convert.ToInt32(tick.Volume)
})
.GroupBy(x => x.Time.RoundDown(Time.OneSecond))
.Select(g => new TradeBar(
g.Key,
symbol,
g.First().LastPrice,
g.Max(t => t.LastPrice),
g.Min(t => t.LastPrice),
g.Last().LastPrice,
g.Sum(t => t.Quantity),
Time.OneSecond))
.Select(tradeBar => new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)))
.ToList();
DataPointCount += result.Count;
return(result);
}
private IEnumerable <Slice> GetHistoryTick(HistoryRequest request, DateTime start, DateTime end)
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.Tick);
if (history == null)
{
return(Enumerable.Empty <Slice>());
}
DataPointCount += history.Count;
return(history
.Select(tick => new Tick
{
Time = tick.Time,
Symbol = symbol,
Value = tick.Price,
TickType = TickType.Trade,
Quantity = Convert.ToInt32(tick.Volume)
})
.Select(tradeBar => new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz))));
}
/// <summary>
/// Transform received data into BaseData object
/// </summary>
/// <param name="e">Received data</param>
/// <param name="requestData">Request information</param>
/// <returns>BaseData object</returns>
private BaseData GetData(LookupEventArgs e, HistoryRequest requestData)
{
var isEquity = requestData.SecurityType == SecurityType.Equity;
var scale = isEquity ? 1000m : 1m;
try
{
switch (e.Type)
{
case LookupType.REQ_HST_TCK:
var t = (LookupTickEventArgs) e;
var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
return new Tick(time, requestData.Symbol, (decimal) t.Last*scale, (decimal) t.Bid*scale, (decimal) t.Ask*scale);
case LookupType.REQ_HST_INT:
var i = (LookupIntervalEventArgs) e;
if (i.DateTimeStamp == DateTime.MinValue) return null;
var istartTime = i.DateTimeStamp - requestData.Resolution.ToTimeSpan();
if (!isEquity) istartTime = istartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
return new TradeBar(istartTime, requestData.Symbol, (decimal) i.Open*scale, (decimal) i.High*scale, (decimal) i.Low*scale, (decimal) i.Close*scale, i.PeriodVolume);
case LookupType.REQ_HST_DWM:
var d = (LookupDayWeekMonthEventArgs) e;
if (d.DateTimeStamp == DateTime.MinValue) return null;
var dstartTime = d.DateTimeStamp - requestData.Resolution.ToTimeSpan();
if (!isEquity) dstartTime = dstartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
return new TradeBar(dstartTime, requestData.Symbol, (decimal) d.Open*scale, (decimal) d.High*scale, (decimal) d.Low*scale, (decimal) d.Close*scale, d.PeriodVolume);
// we don't need to handle these other types
case LookupType.REQ_SYM_SYM:
case LookupType.REQ_SYM_SIC:
case LookupType.REQ_SYM_NAC:
case LookupType.REQ_TAB_MKT:
case LookupType.REQ_TAB_SEC:
case LookupType.REQ_TAB_MKC:
case LookupType.REQ_TAB_SIC:
case LookupType.REQ_TAB_NAC:
default:
return null;
}
}
catch (Exception err)
{
Log.Error("Encountered error while processing request: " + e.Id);
Log.Error(err);
return null;
}
}
/// <summary>
/// Populate request data
/// </summary>
public IEnumerable<Slice> ProcessHistoryRequests(HistoryRequest request)
{
// we can only process equity/forex types here
if (request.SecurityType != SecurityType.Forex && request.SecurityType != SecurityType.Equity)
{
yield break;
}
// Set this process status
_inProgress = true;
var symbol = request.Symbol.Value;
if (request.SecurityType == SecurityType.Forex)
{
symbol = symbol + ".FXCM";
}
var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork);
DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork);
// if we're within a minute of now, don't set the end time
if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1))
{
end = null;
}
Log.Trace(string.Format("HistoryPort.ProcessHistoryJob(): Submitting request: {0}-{1}: {2} {3}->{4}", request.SecurityType, symbol, request.Resolution, start, end ?? DateTime.UtcNow.AddMinutes(-1)));
int id;
var reqid = string.Empty;
switch (request.Resolution)
{
case Resolution.Tick:
id = RequestTickData(symbol, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_TCK, id);
break;
case Resolution.Daily:
id = RequestDailyData(symbol, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_DWM, id);
break;
default:
var interval = new Interval(GetPeriodType(request.Resolution), 1);
id = RequestIntervalData(symbol, interval, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_INT, id);
break;
}
_requestDataByRequestId[reqid] = request;
while (_inProgress)
{
continue;
}
// After all data arrive, we pass it to the algorithm through memory and write to a file
foreach (var key in _currentRequest.Keys)
{
List<BaseData> tradeBars;
if (_currentRequest.TryRemove(key, out tradeBars))
{
foreach (var tradeBar in tradeBars)
{
// Returns IEnumerable<Slice> object
yield return new Slice(tradeBar.EndTime, new[] { tradeBar });
}
}
}
}