/// <summary>
/// Transform received data into BaseData object
/// </summary>
/// <param name="e">Received data</param>
/// <param name="requestData">Request information</param>
/// <returns>BaseData object</returns>
private BaseData GetData(LookupEventArgs e, HistoryRequest requestData)
{
var isEquity = requestData.SecurityType == SecurityType.Equity;
var scale = isEquity ? 1000m : 1m;
try
{
switch (e.Type)
{
case LookupType.REQ_HST_TCK:
var t = (LookupTickEventArgs)e;
var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
return(new Tick(time, requestData.Symbol, (decimal)t.Last * scale, (decimal)t.Bid * scale, (decimal)t.Ask * scale));
case LookupType.REQ_HST_INT:
var i = (LookupIntervalEventArgs)e;
if (i.DateTimeStamp == DateTime.MinValue)
{
return(null);
}
var istartTime = i.DateTimeStamp - requestData.Resolution.ToTimeSpan();
if (!isEquity)
{
istartTime = istartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
}
return(new TradeBar(istartTime, requestData.Symbol, (decimal)i.Open * scale, (decimal)i.High * scale, (decimal)i.Low * scale, (decimal)i.Close * scale, i.PeriodVolume));
case LookupType.REQ_HST_DWM:
var d = (LookupDayWeekMonthEventArgs)e;
if (d.DateTimeStamp == DateTime.MinValue)
{
return(null);
}
var dstartTime = d.DateTimeStamp - requestData.Resolution.ToTimeSpan();
if (!isEquity)
{
dstartTime = dstartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard);
}
return(new TradeBar(dstartTime, requestData.Symbol, (decimal)d.Open * scale, (decimal)d.High * scale, (decimal)d.Low * scale, (decimal)d.Close * scale, d.PeriodVolume));
// we don't need to handle these other types
case LookupType.REQ_SYM_SYM:
case LookupType.REQ_SYM_SIC:
case LookupType.REQ_SYM_NAC:
case LookupType.REQ_TAB_MKT:
case LookupType.REQ_TAB_SEC:
case LookupType.REQ_TAB_MKC:
case LookupType.REQ_TAB_SIC:
case LookupType.REQ_TAB_NAC:
default:
return(null);
}
}
catch (Exception err)
{
Log.Error("Encountered error while processing request: " + e.Id);
Log.Error(err);
return(null);
}
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
/// <summary>
/// Populate request data
/// </summary>
public IEnumerable <Slice> ProcessHistoryRequests(HistoryRequest request)
{
// we can only process equity/forex types here
if (request.SecurityType != SecurityType.Forex && request.SecurityType != SecurityType.Equity)
{
yield break;
}
// Set this process status
_inProgress = true;
var symbol = request.Symbol.Value;
if (request.SecurityType == SecurityType.Forex)
{
symbol = symbol + ".FXCM";
}
var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork);
DateTime?end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork);
// if we're within a minute of now, don't set the end time
if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1))
{
end = null;
}
Log.Trace(string.Format("HistoryPort.ProcessHistoryJob(): Submitting request: {0}-{1}: {2} {3}->{4}", request.SecurityType, symbol, request.Resolution, start, end ?? DateTime.UtcNow.AddMinutes(-1)));
int id;
var reqid = string.Empty;
switch (request.Resolution)
{
case Resolution.Tick:
id = RequestTickData(symbol, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_TCK, id);
break;
case Resolution.Daily:
id = RequestDailyData(symbol, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_DWM, id);
break;
default:
var interval = new Interval(GetPeriodType(request.Resolution), 1);
id = RequestIntervalData(symbol, interval, start, end, true);
reqid = CreateRequestID(LookupType.REQ_HST_INT, id);
break;
}
_requestDataByRequestId[reqid] = request;
while (_inProgress)
{
continue;
}
// After all data arrive, we pass it to the algorithm through memory and write to a file
foreach (var key in _currentRequest.Keys)
{
List <BaseData> tradeBars;
if (_currentRequest.TryRemove(key, out tradeBars))
{
foreach (var tradeBar in tradeBars)
{
// Returns IEnumerable<Slice> object
yield return(new Slice(tradeBar.Time, new[] { tradeBar }));
}
}
}
}
