/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
var futuresUniverseDataCollection = data as FuturesChainUniverseDataCollection;
if (futuresUniverseDataCollection == null)
{
throw new ArgumentException($"Expected data of type '{typeof(FuturesChainUniverseDataCollection).Name}'");
}
var underlying = new Tick {
Time = utcTime
};
// date change detection needs to be done in exchange time zone
if (_cacheDate == data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date)
{
return(Unchanged);
}
var availableContracts = futuresUniverseDataCollection.Data.Select(x => x.Symbol);
var results = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, underlying));
// if results are not dynamic, we cache them and won't call filtering till the end of the day
if (!results.IsDynamic)
{
_cacheDate = data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date;
}
var resultingSymbols = results.ToHashSet();
futuresUniverseDataCollection.FilteredContracts = resultingSymbols;
return(resultingSymbols);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;
if (optionsUniverseDataCollection == null)
{
throw new ArgumentException($"Expected data of type '{typeof(OptionChainUniverseDataCollection).Name}'");
}
// date change detection needs to be done in exchange time zone
if (_cacheDate == data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date)
{
return(Unchanged);
}
var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
var results = Option.ContractFilter.Filter(new OptionFilterUniverse(availableContracts, optionsUniverseDataCollection.Underlying));
// if results are not dynamic, we cache them and won't call filtering till the end of the day
if (!results.IsDynamic)
{
_cacheDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
}
// always prepend the underlying symbol
var resultingSymbols = _underlyingSymbol.Concat(results).ToHashSet();
// we save off the filtered results to the universe data collection for later
// population into the OptionChain. This is non-ideal and could be remedied by
// the universe subscription emitting a special type after selection that could
// be checked for in TimeSlice.Create, but for now this will do
optionsUniverseDataCollection.FilteredContracts = resultingSymbols;
return(resultingSymbols);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
// date change detection needs to be done in exchange time zone
var exchangeDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
if (_cacheDate == exchangeDate)
{
return(Unchanged);
}
var availableContracts = data.Data.Select(x => x.Symbol);
// we will only update unique strikes when there is an exchange date change
_optionFilterUniverse.Refresh(availableContracts, data.Underlying, _lastExchangeDate != exchangeDate);
_lastExchangeDate = exchangeDate;
var results = Option.ContractFilter.Filter(_optionFilterUniverse);
// if results are not dynamic, we cache them and won't call filtering till the end of the day
if (!results.IsDynamic)
{
_cacheDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
}
// always prepend the underlying symbol
return(_underlyingSymbol.Concat(results));
}
/// <summary>
/// Performs universe selection based on the symbol mapping
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">Empty data</param>
/// <returns>The symbols to use</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
yield return(_security.Symbol.Canonical);
var mapFile = _mapFileProvider.ResolveMapFile(new SubscriptionDataConfig(Configuration,
dataMappingMode: UniverseSettings.DataMappingMode,
symbol: _security.Symbol.Canonical));
var mappedSymbol = mapFile.GetMappedSymbol(utcTime.ConvertFromUtc(_security.Exchange.TimeZone));
if (!string.IsNullOrEmpty(mappedSymbol) && mappedSymbol != _mappedSymbol)
{
if (_currentSymbol != null)
{
// let's emit the old and new for the mapping date
yield return(_currentSymbol);
}
_mappedSymbol = mappedSymbol;
_currentSymbol = _security.Symbol.Canonical
.UpdateMappedSymbol(mappedSymbol, Configuration.ContractDepthOffset)
.Underlying;
}
if (_currentSymbol != null)
{
((IContinuousSecurity)_security).Mapped = _currentSymbol;
yield return(_currentSymbol);
}
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;
if (optionsUniverseDataCollection == null)
{
throw new ArgumentException(string.Format("Expected data of type '{0}'", typeof (OptionChainUniverseDataCollection).Name));
}
_underlying = optionsUniverseDataCollection.Underlying ?? _underlying;
optionsUniverseDataCollection.Underlying = _underlying;
if (_underlying == null || data.Data.Count == 0)
{
return Unchanged;
}
var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
var results = _option.ContractFilter.Filter(availableContracts, _underlying).ToHashSet();
// we save off the filtered results to the universe data collection for later
// population into the OptionChain. This is non-ideal and could be remedied by
// the universe subscription emitting a special type after selection that could
// be checked for in TimeSlice.Create, but for now this will do
optionsUniverseDataCollection.FilteredContracts = results;
return results;
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public IEnumerable <Symbol> PerformSelection(DateTime utcTime, BaseDataCollection data)
{
// select empty set of symbols after dispose requested
if (DisposeRequested)
{
OnSelectionChanged();
return(Enumerable.Empty <Symbol>());
}
var result = SelectSymbols(utcTime, data);
if (ReferenceEquals(result, Unchanged))
{
return(Unchanged);
}
var selections = result.ToHashSet();
var hasDiffs = _previousSelections.AreDifferent(selections);
_previousSelections = selections;
if (!hasDiffs)
{
return(Unchanged);
}
OnSelectionChanged(selections);
return(selections);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;
if (optionsUniverseDataCollection == null)
{
throw new ArgumentException(string.Format("Expected data of type '{0}'", typeof(OptionChainUniverseDataCollection).Name));
}
_underlying = optionsUniverseDataCollection.Underlying ?? _underlying;
optionsUniverseDataCollection.Underlying = _underlying;
if (_underlying == null || data.Data.Count == 0)
{
return(Unchanged);
}
var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
var results = _option.ContractFilter.Filter(availableContracts, _underlying).ToHashSet();
// we save off the filtered results to the universe data collection for later
// population into the OptionChain. This is non-ideal and could be remedied by
// the universe subscription emitting a special type after selection that could
// be checked for in TimeSlice.Create, but for now this will do
optionsUniverseDataCollection.FilteredContracts = results;
return(results);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
using (Py.GIL())
{
var symbols = _universe.SelectSymbols(utcTime, data) as PyObject;
var iterator = symbols.GetIterator();
foreach (PyObject symbol in iterator)
{
yield return(symbol.GetAndDispose <Symbol>());
}
iterator.Dispose();
symbols.Dispose();
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable<BaseData> Read(SubscriptionDataSource source)
{
IStreamReader reader = null;
var instances = new BaseDataCollection();
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache);
break;
}
var raw = "";
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
}
}
catch (Exception err)
{
OnReaderError(raw, err);
}
foreach (var instance in instances.Data)
{
yield return instance;
}
}
finally
{
if (reader != null)
reader.Dispose();
}
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public IEnumerable <Symbol> PerformSelection(DateTime utcTime, BaseDataCollection data)
{
var result = SelectSymbols(utcTime, data);
if (ReferenceEquals(result, Unchanged))
{
return(Unchanged);
}
var selections = result.ToHashSet();
var hasDiffs = _previousSelections.Except(selections).Union(selections.Except(_previousSelections)).Any();
_previousSelections = selections;
if (!hasDiffs)
{
return(Unchanged);
}
return(selections);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
var underlying = new Tick {
Time = utcTime
};
// date change detection needs to be done in exchange time zone
if (_cacheDate == data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date)
{
return(Unchanged);
}
var availableContracts = data.Data.Select(x => x.Symbol);
var results = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, underlying));
// if results are not dynamic, we cache them and won't call filtering till the end of the day
if (!results.IsDynamic)
{
_cacheDate = data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date;
}
return(results);
}
/// <summary>
/// Returns the symbols defined by the user for this universe
/// </summary>
/// <param name="utcTime">The curren utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(_selector(utcTime));
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return _selector(data.Data.OfType<FineFundamental>());
}
/// <summary> /// Performs universe selection using the data specified /// </summary> /// <param name="utcTime">The current utc time</param> /// <param name="data">The symbols to remain in the universe</param> /// <returns>The data that passes the filter</returns> public abstract IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data);
/// <summary>
/// Performs an initial, coarse filter
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The coarse fundamental data</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return _universeSelector(data.Data);
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {Symbol = symbol};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
try
{
var tmp = list;
return new List<BaseData> { tmp };
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
return Enumerable.Empty<BaseData>();
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return Universe.SelectSymbols(utcTime, data);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.SubscriptionSettings;
var limit = 1000; //daily/hourly limit
var resolution = settings.Resolution;
switch (resolution)
{
case Resolution.Tick:
limit = _algorithm.Securities.TickLimit;
break;
case Resolution.Second:
limit = _algorithm.Securities.SecondLimit;
break;
case Resolution.Minute:
limit = _algorithm.Securities.MinuteLimit;
break;
}
// subtract current subscriptions that can't be removed
limit -= _algorithm.Securities.Count(x => x.Value.Resolution == resolution && x.Value.HoldStock);
if (limit < 1)
{
// if we don't have room for more securities then we can't really do anything here.
_algorithm.Error("Unable to add more securities from universe selection due to holding stock.");
return SecurityChanges.None;
}
// perform initial filtering and limit the result
var selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData.Data);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return SecurityChanges.None;
}
var selections = selectSymbolsResult.Take(limit).ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List<Security>();
var removals = new List<Security>();
// determine which data subscriptions need to be removed for this market
foreach (var subscription in _dataFeed.Subscriptions)
{
// universes can only remove members of their own
if (!universe.ContainsMember(subscription.Security)) continue;
// never remove universe selection subscriptions
if (subscription.IsUniverseSelectionSubscription) continue;
var config = subscription.Configuration;
// never remove internal feeds
if (config.IsInternalFeed) continue;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol)) continue;
// let the algorithm know this security has been removed from the universe
removals.Add(subscription.Security);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!subscription.Security.HoldStock && !openOrders.Any())
{
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
subscription.Security.Cache.Reset();
if (_dataFeed.RemoveSubscription(subscription))
{
universe.RemoveMember(subscription.Security);
}
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol)) continue;
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(security);
}
}
// return None if there's no changes, otherwise return what we've modified
return additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(_selector(data.Data.OfType <CoarseFundamental>()));
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.UniverseSettings;
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return SecurityChanges.None;
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List<Security>();
var removals = new List<Security>();
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
var config = member.SubscriptionDataConfig;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol)) continue;
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member)) continue;
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
_dataFeed.RemoveSubscription(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol)) continue;
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(settings.Resolution, out reason))
{
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason);
break;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(dateTimeUtc, security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
}
// return None if there's no changes, otherwise return what we've modified
return additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is UserDefinedUniverse)
{
// Trigger universe selection when security added/removed after Initialize
var universe = (UserDefinedUniverse) request.Universe;
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if (symbol == null) return;
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
return new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder());
}
if (request.Configuration.Type == typeof (CoarseFundamental))
{
return new BaseDataCollectionSubscriptionEnumeratorFactory();
}
if (request.Configuration.Type == typeof(FineFundamental))
{
return new FineFundamentalSubscriptionEnumeratorFactory();
}
if (request.Universe is OptionChainUniverse)
{
return new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
}
}
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return new PostCreateConfigureSubscriptionEnumeratorFactory(
new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true),
enumerator => ConfigureEnumerator(request, false, enumerator)
);
}
/// <summary>
/// Performs an initial, coarse filter
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The coarse fundamental data</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(_universeSelector(data.Data.Cast <T>()));
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if (symbol == null) return;
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return subscription;
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return SecurityChanges.None;
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List<Security>();
var removals = new List<Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol)) continue;
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member)) continue;
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc))
{
if (subscription.IsUniverseSubscription)
{
removals.Remove(member);
}
else
{
_dataFeed.RemoveSubscription(subscription.Configuration);
}
}
// remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD!
SymbolCache.TryRemove(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedMember = universe.AddMember(dateTimeUtc, security);
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = addedMember;
}
}
if (addedSubscription)
{
additions.Add(security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return securityChanges;
}
/// <summary>
/// Returns the symbols defined by the user for this universe
/// </summary>
/// <param name="utcTime">The curren utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return _selector(utcTime);
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(Universe.SelectSymbols(utcTime, data));
}
/// <summary>
/// Syncs the specifies subscriptions at the frontier time
/// </summary>
/// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="sliceTimeZone">The time zone of the created slice object</param>
/// <param name="cashBook">The cash book, used for creating the cash book updates</param>
/// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier.
/// This value will equal DateTime.MaxValue when the subscriptions are all finished</param>
/// <returns>A time slice for the specified frontier time</returns>
public TimeSlice Sync(DateTime frontier, IEnumerable<Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier)
{
var changes = SecurityChanges.None;
nextFrontier = DateTime.MaxValue;
var earlyBirdTicks = nextFrontier.Ticks;
var data = new List<DataFeedPacket>();
var universeData = new Dictionary<Universe, BaseDataCollection>();
SecurityChanges newChanges;
do
{
universeData.Clear();
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
var packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
data.Add(packet);
var configuration = subscription.Configuration;
var offsetProvider = subscription.OffsetProvider;
var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier);
while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks)
{
// we want bars rounded using their subscription times, we make a clone
// so we don't interfere with the enumerator's internal logic
var clone = subscription.Current.Clone(subscription.Current.IsFillForward);
clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours);
packet.Add(clone);
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
break;
}
}
// we have new universe data to select based on, store the subscription data until the end
if (subscription.IsUniverseSelectionSubscription && packet.Count > 0)
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (!universeData.TryGetValue(subscription.Universe, out collection))
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = subscription.Current as OptionChainUniverseDataCollection;
var underlying = current != null ? current.Underlying : null;
collection = new OptionChainUniverseDataCollection(frontier, subscription.Configuration.Symbol, packetData, underlying);
}
else
{
collection = new BaseDataCollection(frontier, subscription.Configuration.Symbol, packetData);
}
universeData[subscription.Universe] = collection;
}
else
{
collection.Data.AddRange(packetData);
}
}
if (subscription.Current != null)
{
// take the earliest between the next piece of data or the next tz discontinuity
earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity()));
}
}
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontier, baseDataCollection);
}
changes += newChanges;
}
while (newChanges != SecurityChanges.None);
nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
return TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes);
}