public void FundamentalUniverseSelectionModelCanBeInherited()
{
var code = @"
from AlgorithmImports import *
from Selection.FundamentalUniverseSelectionModel import FundamentalUniverseSelectionModel
class MockUniverseSelectionModel(FundamentalUniverseSelectionModel):
def __init__(self):
super().__init__(False)
def SelectCoarse(self, algorithm, coarse):
return [Symbol.Create('SPY', SecurityType.Equity, Market.USA)]";
using (Py.GIL())
{
dynamic pyModel = PythonEngine
.ModuleFromString(Guid.NewGuid().ToString(), code)
.GetAttr("MockUniverseSelectionModel");
var model = new UniverseSelectionModelPythonWrapper(pyModel());
var universes = model.CreateUniverses(new QCAlgorithm()).ToList();
Assert.AreEqual(1, universes.Count);
var data = new BaseDataCollection();
data.Add(new CoarseFundamental());
var universe = universes.First();
var symbols = universe.SelectSymbols(DateTime.Now, data).ToList();
Assert.AreEqual(1, symbols.Count);
var expected = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
var symbol = symbols.First();
Assert.AreEqual(expected, symbol);
}
}
void IEmployeesModelService.SaveEmployee(EmployeeEntity entity)
{
if (entity.Id == 0)
{
_employess.Add(new EmployeeModel(entity));
}
_employessRepository.SaveEmployee(entity);
}
public void SaveDepartment(DepartmentEntity entity)
{
if (entity.Id == 0)
{
_departments.Add(new DepartmentModel(entity));
}
_departmentsRepository.SaveDepartment(entity);
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
foreach (var point in base.Read(source))
{
if (point is BaseDataCollection)
{
// if underlying already is returning a collection let it through as is
yield return(point);
}
else
{
if (_collection != null && _collection.EndTime != point.EndTime)
{
// when we get a new time we flush current collection instance, if any
yield return(_collection);
_collection = null;
}
if (_collection == null)
{
_collection = (BaseDataCollection)Activator.CreateInstance(_collectionType);
_collection.Time = point.Time;
_collection.Symbol = point.Symbol;
_collection.EndTime = point.EndTime;
}
// aggregate the data points
_collection.Add(point);
}
}
// underlying reader ended, flush current collection instance if any
if (_collection != null)
{
yield return(_collection);
_collection = null;
}
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null
// if the universe has been disposed we don't perform selection. This us handled bellow by 'Universe.PerformSelection'
// but in this case we directly call 'SelectSymbols' because we want to perform fine selection even if coarse returns the same
// symbols, see 'Universe.PerformSelection', which detects this and returns 'Universe.Unchanged'
&& !universe.DisposeRequested)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
// Create a dictionary of CoarseFundamental keyed by Symbol that also has FineFundamental
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var allCoarse = universeData.Data.OfType <CoarseFundamental>();
var coarseData = allCoarse.Where(c => c.HasFundamentalData)
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
// Remove selected symbols that does not have fine fundamental data
var anyDoesNotHaveFundamentalData = false;
// only pre filter selected symbols if there actually is any coarse data. This way we can support custom universe filtered by fine fundamental data
// which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem,
// 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it
if (allCoarse.Any())
{
selectSymbolsResult = selectSymbolsResult
.Where(
symbol =>
{
var result = coarseData.ContainsKey(symbol);
anyDoesNotHaveFundamentalData |= !result;
return(result);
}
);
}
if (!_anyDoesNotHaveFundamentalDataWarningLogged && anyDoesNotHaveFundamentalData)
{
_algorithm.Debug("Note: Your coarse selection filter was updated to exclude symbols without fine fundamental data. Make sure your coarse filter excludes symbols where HasFundamental is false.");
_anyDoesNotHaveFundamentalDataWarningLogged = true;
}
// use all available threads, the entire system is waiting for this to complete
var options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(selectSymbolsResult, options, symbol =>
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = _securityService.CreateSecurity(symbol,
config,
addToSymbolCache: false);
var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, _dataProvider))
{
if (enumerator.MoveNext())
{
lock (fineCollection.Data)
{
fineCollection.Add(enumerator.Current);
}
}
}
});
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
AssetClassification = fine.AssetClassification,
CompanyProfile = fine.CompanyProfile,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios,
Market = fine.Symbol.ID.Market
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
// set the fine fundamental price property to yesterday's closing price
fine.Value = coarse.Value;
}
universeData.Add(fundamentals);
}
// END -- HACK ATTACK -- END
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
// first check for no pending removals, even if the universe selection
// didn't change we might need to remove a security because a position was closed
RemoveSecurityFromUniverse(
_pendingRemovalsManager.CheckPendingRemovals(selections, universe),
dateTimeUtc,
algorithmEndDateUtc);
// check for no changes second
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Securities.Values.OrderBy(member => member.Security.Symbol.SecurityType))
{
var security = member.Security;
// if we've selected this subscription again, keep it
if (selections.Contains(security.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, security))
{
continue;
}
_securityChangesConstructor.Remove(member.Security, member.IsInternal);
RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe),
dateTimeUtc,
algorithmEndDateUtc);
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
if (universe.Securities.ContainsKey(symbol))
{
// if its already part of the universe no need to re add it
continue;
}
Security underlying = null;
if (symbol.HasUnderlying)
{
underlying = GetOrCreateSecurity(pendingAdditions, symbol.Underlying, universe.UniverseSettings);
}
// create the new security, the algorithm thread will add this at the appropriate time
var security = GetOrCreateSecurity(pendingAdditions, symbol, universe.UniverseSettings, underlying);
var addedSubscription = false;
var dataFeedAdded = false;
var internalFeed = true;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
_algorithm.SubscriptionManager.SubscriptionDataConfigService))
{
if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw.
!security.Subscriptions.Contains(request.Configuration))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
security.AddData(request.Configuration);
}
var toRemove = _currencySubscriptionDataConfigManager.GetSubscriptionDataConfigToRemove(request.Configuration.Symbol);
if (toRemove != null)
{
Log.Trace($"UniverseSelection.ApplyUniverseSelection(): Removing internal currency data feed {toRemove}");
_dataManager.RemoveSubscription(toRemove);
}
// 'dataFeedAdded' will help us notify the user for security changes only once per non internal subscription
// for example two universes adding the sample configuration, we don't want two notifications
dataFeedAdded = _dataManager.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
// if any config isn't internal then it's not internal
internalFeed &= request.Configuration.IsInternalFeed;
_internalSubscriptionManager.AddedSubscriptionRequest(request);
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security, internalFeed);
if (addedMember && dataFeedAdded)
{
_securityChangesConstructor.Add(security, internalFeed);
}
}
}
var securityChanges = _securityChangesConstructor.Flush();
// Add currency data feeds that weren't explicitly added in Initialize
if (securityChanges.AddedSecurities.Count > 0)
{
EnsureCurrencyDataFeeds(securityChanges);
}
if (securityChanges != SecurityChanges.None && Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}