/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable <Slice> GetHistory(IEnumerable <HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
// create subscription objects from the configs
var subscriptions = new List <Subscription>();
foreach (var request in requests)
{
var history = _brokerage.GetHistory(request);
var subscription = CreateSubscription(request, history);
_dataPermissionManager.AssertConfiguration(subscription.Configuration, request.StartTimeLocal, request.EndTimeLocal);
subscriptions.Add(subscription);
}
return(CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone));
}
/// <summary>
/// Adds a new <see cref="Subscription"/> to provide data for the specified security.
/// </summary>
/// <param name="request">Defines the <see cref="SubscriptionRequest"/> to be added</param>
/// <returns>True if the subscription was created and added successfully, false otherwise</returns>
public bool AddSubscription(SubscriptionRequest request)
{
// guarantee the configuration is present in our config collection
// this is related to GH issue 3877: where we added a configuration which we also removed
_subscriptionManagerSubscriptions.TryAdd(request.Configuration, request.Configuration);
Subscription subscription;
if (DataFeedSubscriptions.TryGetValue(request.Configuration, out subscription))
{
// duplicate subscription request
subscription.AddSubscriptionRequest(request);
// only result true if the existing subscription is internal, we actually added something from the users perspective
return(subscription.Configuration.IsInternalFeed);
}
// before adding the configuration to the data feed let's assert it's valid
_dataPermissionManager.AssertConfiguration(request.Configuration, request.StartTimeLocal, request.EndTimeLocal);
subscription = _dataFeed.CreateSubscription(request);
if (subscription == null)
{
Log.Trace($"DataManager.AddSubscription(): Unable to add subscription for: {request.Configuration}");
// subscription will be null when there's no tradeable dates for the security between the requested times, so
// don't even try to load the data
return(false);
}
if (_liveMode)
{
OnSubscriptionAdded(subscription);
Log.Trace($"DataManager.AddSubscription(): Added {request.Configuration}." +
$" Start: {request.StartTimeUtc}. End: {request.EndTimeUtc}");
}
else if (Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
Log.Debug($"DataManager.AddSubscription(): Added {request.Configuration}." +
$" Start: {request.StartTimeUtc}. End: {request.EndTimeUtc}");
}
return(DataFeedSubscriptions.TryAdd(subscription));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}