public void SecurityManagerCanCreate_ForexOrCrypto_WithCorrectSubscriptions(string ticker, SecurityType type, string market)
{
var symbol = Symbol.Create(ticker, type, market);
var marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, type);
var defaultQuoteCurrency = symbol.Value.Substring(3);
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, defaultQuoteCurrency);
var actual = SecurityManager.CreateSecurity(typeof(QuoteBar),
_securityPortfolioManager,
_subscriptionManager,
marketHoursDbEntry.ExchangeHours,
marketHoursDbEntry.DataTimeZone,
symbolProperties,
_securityInitializer,
symbol,
Resolution.Second,
false,
1.0m,
false,
false,
false,
false);
Assert.AreEqual(actual.Subscriptions.Count(), 1);
Assert.AreEqual(actual.Subscriptions.First().Type, typeof(QuoteBar));
Assert.AreEqual(actual.Subscriptions.First().TickType, TickType.Quote);
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data types to use.
/// If the config already existed will return existing instance instead
/// </summary>
public List <SubscriptionDataConfig> Add(
Symbol symbol,
Resolution resolution,
bool fillForward,
bool extendedMarketHours,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
List <Tuple <Type, TickType> > subscriptionDataTypes = null
)
{
var dataTypes = subscriptionDataTypes ??
LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution, symbol.IsCanonical());
var marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
var exchangeHours = marketHoursDbEntry.ExchangeHours;
var dataNormalizationMode = symbol.ID.SecurityType == SecurityType.Option || symbol.ID.SecurityType == SecurityType.Future
? DataNormalizationMode.Raw
: DataNormalizationMode.Adjusted;
if (marketHoursDbEntry.DataTimeZone == null)
{
throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
"DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
}
if (exchangeHours.TimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
"ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
}
if (!dataTypes.Any())
{
throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
}
var result = (from subscriptionDataType in dataTypes
let dataType = subscriptionDataType.Item1
let tickType = subscriptionDataType.Item2
select new SubscriptionDataConfig(
dataType,
symbol,
resolution,
marketHoursDbEntry.DataTimeZone,
exchangeHours.TimeZone,
fillForward,
extendedMarketHours,
isInternalFeed,
isCustomData,
isFilteredSubscription: isFilteredSubscription,
tickType: tickType,
dataNormalizationMode: dataNormalizationMode)).ToList();
for (int i = 0; i < result.Count; i++)
{
result[i] = SubscriptionManagerGetOrAdd(result[i]);
}
return(result);
}
/// <summary>
/// Creates the universes for this algorithm.
/// Called at algorithm start.
/// </summary>
/// <returns>The universes defined by this model</returns>
public override IEnumerable <Universe> CreateUniverses(QCAlgorithmFramework algorithm)
{
var universeSettings = _universeSettings ?? algorithm.UniverseSettings;
var securityInitializer = _securityInitializer ?? algorithm.SecurityInitializer;
var resolution = universeSettings.Resolution;
var type = resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar);
// universe per security type/market
foreach (var grp in _symbols.GroupBy(s => new { s.ID.Market, s.SecurityType }))
{
MarketHoursDatabase.Entry entry;
var market = grp.Key.Market;
var securityType = grp.Key.SecurityType;
var universeSymbol = Symbol.Create($"manual-universe-selection-model-{securityType}-{market}", securityType, market);
if (securityType == SecurityType.Base)
{
// add an entry for this custom universe symbol -- we don't really know the time zone for sure,
// but we set it to TimeZones.NewYork in AddData, also, since this is a manual universe, the time
// zone doesn't actually matter since this universe specifically doesn't do anything with data.
var symbolString = MarketHoursDatabase.GetDatabaseSymbolKey(universeSymbol);
var alwaysOpen = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
entry = MarketHours.SetEntry(market, symbolString, securityType, alwaysOpen, TimeZones.NewYork);
}
else
{
entry = MarketHours.GetEntry(market, (string)null, securityType);
}
var config = new SubscriptionDataConfig(type, universeSymbol, resolution, entry.DataTimeZone, entry.ExchangeHours.TimeZone, false, false, true);
yield return(new ManualUniverse(config, universeSettings, grp));
}
}
private IEnumerable <SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution?resolution = null)
{
Security security;
if (Securities.TryGetValue(symbol, out security))
{
// find all subscriptions matching the requested type with a higher resolution than requested
return(from sub in security.Subscriptions.OrderByDescending(s => s.Resolution)
where type.IsAssignableFrom(sub.Type)
select sub);
}
else
{
var entry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
resolution = GetResolution(symbol, resolution);
return(SubscriptionManager
.LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution.Value, symbol.IsCanonical())
.Select(x => new SubscriptionDataConfig(
x.Item1,
symbol,
resolution.Value,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
UniverseSettings.FillForward,
UniverseSettings.ExtendedMarketHours,
true,
false,
x.Item2,
true,
UniverseSettings.DataNormalizationMode)));
}
}
public void SecurityManagerCanCreate_Forex_WithCorrectSubscriptions()
{
var forexSymbol = Symbol.Create("EURUSD", SecurityType.Forex, Market.FXCM);
var forexMarketHoursDbEntry = _marketHoursDatabase.GetEntry(forexSymbol.ID.Market, forexSymbol, SecurityType.Forex);
var forexDefaultQuoteCurrency = forexSymbol.Value.Substring(3);
var forexSymbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(forexSymbol.ID.Market, forexSymbol, forexSymbol.ID.SecurityType, forexDefaultQuoteCurrency);
var subscriptionTypes = new List <Type>()
{
typeof(QuoteBar)
};
var forex = SecurityManager.CreateSecurity(subscriptionTypes,
_securityPortfolioManager,
_subscriptionManager,
forexMarketHoursDbEntry.ExchangeHours,
forexMarketHoursDbEntry.DataTimeZone,
forexSymbolProperties,
_securityInitializer,
forexSymbol,
Resolution.Second,
false,
1.0m,
false,
false,
false,
false);
Assert.AreEqual(forex.Subscriptions.Count(), 1);
Assert.AreEqual(forex.Subscriptions.First().Type, typeof(QuoteBar));
Assert.AreEqual(forex.Subscriptions.First().TickType, TickType.Quote);
}
public Universe AddUniverse <T>(SecurityType securityType, string name, Resolution resolution, string market, UniverseSettings universeSettings, Func <IEnumerable <T>, IEnumerable <Symbol> > selector)
{
var marketHoursDbEntry = MarketHoursDatabase.GetEntry(market, name, securityType);
var dataTimeZone = marketHoursDbEntry.DataTimeZone;
var exchangeTimeZone = marketHoursDbEntry.ExchangeHours.TimeZone;
var symbol = QuantConnect.Symbol.Create(name, securityType, market, baseDataType: typeof(T));
var config = new SubscriptionDataConfig(typeof(T), symbol, resolution, dataTimeZone, exchangeTimeZone, false, false, true, true, isFilteredSubscription: false);
return(AddUniverse(new FuncUniverse(config, universeSettings, d => selector(d.OfType <T>()))));
}
public Universe AddUniverse(SecurityType securityType, string name, Resolution resolution, string market, UniverseSettings universeSettings, Func <DateTime, IEnumerable <string> > selector)
{
var marketHoursDbEntry = MarketHoursDatabase.GetEntry(market, name, securityType);
var dataTimeZone = marketHoursDbEntry.DataTimeZone;
var exchangeTimeZone = marketHoursDbEntry.ExchangeHours.TimeZone;
var symbol = QuantConnect.Symbol.Create(name, securityType, market);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, resolution, dataTimeZone, exchangeTimeZone, false, false, true, isFilteredSubscription: false);
return(AddUniverse(new UserDefinedUniverse(config, universeSettings, resolution.ToTimeSpan(), selector)));
}
/// <summary>
/// Expiry function for CBOT Futures Options entries.
/// Returns the Friday before the 2nd last business day of the month preceding the future contract expiry month.
/// </summary>
/// <param name="underlyingFuture">Underlying future symbol</param>
/// <param name="expiryMonth">Expiry month date</param>
/// <returns>Expiry DateTime of the Future Option</returns>
private static DateTime FridayBeforeTwoBusinessDaysBeforeEndOfMonth(Symbol underlyingFuture, DateTime expiryMonth)
{
var holidays = _mhdb.GetEntry(underlyingFuture.ID.Market, underlyingFuture, SecurityType.Future)
.ExchangeHours
.Holidays;
var expiryMonthPreceding = expiryMonth.AddMonths(-1).AddDays(-(expiryMonth.Day - 1));
var fridayBeforeSecondLastBusinessDay = FuturesExpiryUtilityFunctions.NthLastBusinessDay(
expiryMonthPreceding,
2,
holidays).AddDays(-1);
while (fridayBeforeSecondLastBusinessDay.DayOfWeek != DayOfWeek.Friday)
{
fridayBeforeSecondLastBusinessDay = FuturesExpiryUtilityFunctions.AddBusinessDays(fridayBeforeSecondLastBusinessDay, -1, false, holidays);
}
return(fridayBeforeSecondLastBusinessDay);
}
private SecurityExchangeHours GetExchangeHours(Symbol symbol)
{
Security security;
if (Securities.TryGetValue(symbol, out security))
{
return(security.Exchange.Hours);
}
return(MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours);
}
/// <summary>
/// Creates a new universe and adds it to the algorithm
/// </summary>
/// <param name="dataType">The data type</param>
/// <param name="securityType">The security type the universe produces</param>
/// <param name="name">A unique name for this universe</param>
/// <param name="resolution">The epected resolution of the universe data</param>
/// <param name="market">The market for selected symbols</param>
/// <param name="universeSettings">The subscription settings to use for newly created subscriptions</param>
/// <param name="pySelector">Function delegate that performs selection on the universe data</param>
public void AddUniverse(Type dataType, SecurityType securityType, string name, Resolution resolution, string market, UniverseSettings universeSettings, PyObject pySelector)
{
var marketHoursDbEntry = MarketHoursDatabase.GetEntry(market, name, securityType);
var dataTimeZone = marketHoursDbEntry.DataTimeZone;
var exchangeTimeZone = marketHoursDbEntry.ExchangeHours.TimeZone;
var symbol = QuantConnect.Symbol.Create(name, securityType, market);
var config = new SubscriptionDataConfig(dataType, symbol, resolution, dataTimeZone, exchangeTimeZone, false, false, true, true, isFilteredSubscription: false);
var selector = PythonUtil.ToFunc <IEnumerable <IBaseData>, object[]>(pySelector);
AddUniverse(new FuncUniverse(config, universeSettings, SecurityInitializer, d => selector(d)
.Select(x => x is Symbol ? (Symbol)x : QuantConnect.Symbol.Create((string)x, securityType, market))));
}
private IEnumerable <SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution?resolution = null)
{
var matchingSubscriptions = SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
// find all subscriptions matching the requested type with a higher resolution than requested
.OrderByDescending(s => s.Resolution)
// lets make sure to respect the order of the data types
.ThenByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType))
.ThenBy(config => config.IsInternalFeed ? 1 : 0)
.Where(s => SubscriptionDataConfigTypeFilter(type, s.Type))
.ToList();
// we use the subscription manager registered configurations here, we can not rely on the Securities collection
// since this might be called when creating a security and warming it up
if (matchingSubscriptions.Count != 0)
{
if (resolution.HasValue &&
(resolution == Resolution.Daily || resolution == Resolution.Hour) &&
symbol.SecurityType == SecurityType.Equity)
{
// for Daily and Hour resolution, for equities, we have to
// filter out any existing subscriptions that could be of Quote type
// This could happen if they were Resolution.Minute/Second/Tick
return(matchingSubscriptions.Where(s => s.TickType != TickType.Quote));
}
return(matchingSubscriptions);
}
else
{
var entry = MarketHoursDatabase.GetEntry(symbol, new [] { type });
resolution = GetResolution(symbol, resolution);
return(SubscriptionManager
.LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution.Value, symbol.IsCanonical())
.Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
.Select(x => new SubscriptionDataConfig(
x.Item1,
symbol,
resolution.Value,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
UniverseSettings.FillForward,
UniverseSettings.ExtendedMarketHours,
true,
false,
x.Item2,
true,
UniverseSettings.DataNormalizationMode)));
}
}
private IEnumerable <TradingDay> PopulateTradingDays(DateTime start, DateTime end)
{
var symbols = _securityManager.Keys;
var holidays = new HashSet <DateTime>();
foreach (var symbol in symbols)
{
var entry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
foreach (var holiday in entry.ExchangeHours.Holidays)
{
holidays.Add(holiday.Date);
}
}
var qlCalendar = new UnitedStates();
var options = symbols.Where(x => x.ID.SecurityType.IsOption()).ToList();
var futures = symbols.Where(x => x.ID.SecurityType == SecurityType.Future).ToList();
var totalDays = (int)(end.Date.AddDays(1.0) - start.Date).TotalDays;
if (totalDays < 0)
{
throw new ArgumentException(
$"TradingCalendar.PopulateTradingDays(): Total days is negative ({totalDays}), indicating reverse start and end times." +
$" Check your usage of TradingCalendar to ensure proper arrangement of variables");
}
foreach (var dayIdx in Enumerable.Range(0, totalDays))
{
var currentDate = start.Date.AddDays(dayIdx);
var publicHoliday = holidays.Contains(currentDate) || !qlCalendar.isBusinessDay(currentDate);
var weekend = currentDate.DayOfWeek == DayOfWeek.Sunday ||
currentDate.DayOfWeek == DayOfWeek.Saturday;
var businessDay = !publicHoliday && !weekend;
yield return
(new TradingDay
{
Date = currentDate,
PublicHoliday = publicHoliday,
Weekend = weekend,
BusinessDay = businessDay,
OptionExpirations = options.Where(x => x.ID.Date.Date == currentDate),
FutureExpirations = futures.Where(x => x.ID.Date.Date == currentDate)
});
}
}
private MarketHoursDatabase.Entry GetMarketHours(Symbol symbol)
{
var hoursEntry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
// user can override the exchange hours in algorithm, i.e. HistoryAlgorithm
Security security;
if (Securities.TryGetValue(symbol, out security))
{
return(new MarketHoursDatabase.Entry(hoursEntry.DataTimeZone, security.Exchange.Hours));
}
return(hoursEntry);
}
/// <summary>
/// AddData a new user defined data source, requiring only the minimum config options.
/// </summary>
/// <param name="dataType">Data source type</param>
/// <param name="symbol">Key/Symbol for data</param>
/// <param name="resolution">Resolution of the Data Required</param>
/// <param name="timeZone">Specifies the time zone of the raw data</param>
/// <param name="fillDataForward">When no data available on a tradebar, return the last data that was generated</param>
/// <param name="leverage">Custom leverage per security</param>
/// <returns>The new <see cref="Security"/></returns>
public Security AddData(Type dataType, string symbol, Resolution resolution, DateTimeZone timeZone, bool fillDataForward = false, decimal leverage = 1.0m)
{
var marketHoursDbEntry = MarketHoursDatabase.GetEntry(Market.USA, symbol, SecurityType.Base, timeZone);
//Add this to the data-feed subscriptions
var symbolObject = new Symbol(SecurityIdentifier.GenerateBase(symbol, Market.USA), symbol);
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(Market.USA, symbol, SecurityType.Base, CashBook.AccountCurrency);
//Add this new generic data as a tradeable security:
var security = SecurityManager.CreateSecurity(dataType, Portfolio, SubscriptionManager, marketHoursDbEntry.ExchangeHours, marketHoursDbEntry.DataTimeZone,
symbolProperties, SecurityInitializer, symbolObject, resolution, fillDataForward, leverage, true, false, true, LiveMode);
AddToUserDefinedUniverse(security);
return(security);
}
private IEnumerable <SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution?resolution = null)
{
Security security;
if (Securities.TryGetValue(symbol, out security))
{
// find all subscriptions matching the requested type with a higher resolution than requested
var matchingSubscriptions = from sub in security.Subscriptions.OrderByDescending(s => s.Resolution)
where SubscriptionDataConfigTypeFilter(type, sub.Type)
select sub;
if (resolution.HasValue &&
(resolution == Resolution.Daily || resolution == Resolution.Hour) &&
symbol.SecurityType == SecurityType.Equity)
{
// for Daily and Hour resolution, for equities, we have to
// filter out any existing subscriptions that could be of Quote type
// This could happen if they were Resolution.Minute/Second/Tick
matchingSubscriptions = matchingSubscriptions.Where(s => s.TickType != TickType.Quote);
}
return(matchingSubscriptions);
}
else
{
var entry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
resolution = GetResolution(symbol, resolution);
return(SubscriptionManager
.LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution.Value, symbol.IsCanonical())
.Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
.Select(x => new SubscriptionDataConfig(
x.Item1,
symbol,
resolution.Value,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
UniverseSettings.FillForward,
UniverseSettings.ExtendedMarketHours,
true,
false,
x.Item2,
true,
UniverseSettings.DataNormalizationMode)));
}
}
/// <summary>
/// Creates the universes for this algorithm. Called at algorithm start.
/// </summary>
/// <returns>The universes defined by this model</returns>
public override IEnumerable <Universe> CreateUniverses(QCAlgorithm algorithm)
{
var universeSettings = _universeSettings ?? algorithm.UniverseSettings;
var entry = MarketHours.GetEntry(_symbol.ID.Market, (string)null, _symbol.SecurityType);
var config = new SubscriptionDataConfig(
universeSettings.Resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar),
_symbol,
universeSettings.Resolution,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
universeSettings.FillForward,
universeSettings.ExtendedMarketHours,
true
);
yield return(new CustomUniverse(config, universeSettings, _interval, dt => Select(algorithm, dt)));
}
private IEnumerable <TradingDay> PopulateTradingDays(DateTime start, DateTime end)
{
var symbols = _securityManager.Keys;
var holidays = new HashSet <DateTime>();
foreach (var symbol in symbols)
{
var entry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
foreach (var holiday in entry.ExchangeHours.Holidays)
{
holidays.Add(holiday.Date);
}
}
var qlCalendar = new UnitedStates();
var options = symbols.Where(x => x.ID.SecurityType == SecurityType.Option || x.ID.SecurityType == SecurityType.FutureOption).ToList();
var futures = symbols.Where(x => x.ID.SecurityType == SecurityType.Future).ToList();
foreach (var dayIdx in Enumerable.Range(0, (int)(end.Date.AddDays(1.0) - start.Date).TotalDays))
{
var currentDate = start.Date.AddDays(dayIdx);
var publicHoliday = holidays.Contains(currentDate) || !qlCalendar.isBusinessDay(currentDate);
var weekend = currentDate.DayOfWeek == DayOfWeek.Sunday ||
currentDate.DayOfWeek == DayOfWeek.Saturday;
var businessDay = !publicHoliday && !weekend;
yield return
(new TradingDay
{
Date = currentDate,
PublicHoliday = publicHoliday,
Weekend = weekend,
BusinessDay = businessDay,
OptionExpirations = options.Where(x => x.ID.Date.Date == currentDate),
FutureExpirations = futures.Where(x => x.ID.Date.Date == currentDate)
});
}
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
foreach (var symbol in selectSymbolsResult)
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var exchangeHours = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours;
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, CashBook.AccountCurrency);
var quoteCash = _algorithm.Portfolio.CashBook[symbolProperties.QuoteCurrency];
var security = new Equity(symbol, exchangeHours, quoteCash, symbolProperties);
var localStartTime = dateTimeUtc.ConvertFromUtc(exchangeHours.TimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, dataProvider))
{
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var coarseData = universeData.Data.OfType <CoarseFundamental>()
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Market = coarse.Market;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
}
universeData.Data.Add(fundamentals);
}
// END -- HACK ATTACK -- END
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
var keys = _pendingSecurityAdditions.Keys;
if (keys.Any() && keys.Single() != dateTimeUtc)
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
pendingAdditions.Add(symbol, security);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets, securityChanges);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, new SubscriptionDataConfig(security.Subscriptions.First()), dateTimeUtc, algorithmEndDateUtc));
}
}
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var exchangeHours = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours;
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, CashBook.AccountCurrency);
var quoteCash = _algorithm.Portfolio.CashBook[symbolProperties.QuoteCurrency];
var security = new Equity(symbol, exchangeHours, quoteCash, symbolProperties);
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), dateTimeUtc, dateTimeUtc);
using (var enumerator = factory.CreateEnumerator(request, dataProvider))
{
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
var keys = _pendingSecurityAdditions.Keys;
if (keys.Any() && keys.Single() != dateTimeUtc)
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
pendingAdditions.Add(symbol, security);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets, securityChanges);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, new SubscriptionDataConfig(security.Subscriptions.First()), dateTimeUtc, algorithmEndDateUtc));
}
}
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
private MarketHoursDatabase.Entry GetMarketHours(Symbol symbol)
{
return(MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType));
}
/// <summary>
/// Defines the future chain universe filter
/// </summary>
protected override FutureFilterUniverse Filter(FutureFilterUniverse filter)
{
return(filter.Contracts(FilterByOpenInterest(filter.ToDictionary(x => x, x => _marketHoursDatabase.GetEntry(x.ID.Market, x, x.ID.SecurityType).ExchangeHours))));
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data types to use.
/// If the config already existed will return existing instance instead
/// </summary>
public List <SubscriptionDataConfig> Add(
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
List <Tuple <Type, TickType> > subscriptionDataTypes = null,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
)
{
var dataTypes = subscriptionDataTypes ??
LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution ?? Resolution.Minute, symbol.IsCanonical());
if (!dataTypes.Any())
{
throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
}
var resolutionWasProvided = resolution.HasValue;
foreach (var typeTuple in dataTypes)
{
var baseInstance = typeTuple.Item1.GetBaseDataInstance();
baseInstance.Symbol = symbol;
if (!resolutionWasProvided)
{
var defaultResolution = baseInstance.DefaultResolution();
if (resolution.HasValue && resolution != defaultResolution)
{
// we are here because there are multiple 'dataTypes'.
// if we get different default resolutions lets throw, this shouldn't happen
throw new InvalidOperationException(
$"Different data types ({string.Join(",", dataTypes.Select(tuple => tuple.Item1))})" +
$" provided different default resolutions {defaultResolution} and {resolution}, this is an unexpected invalid operation.");
}
resolution = defaultResolution;
}
else
{
// only assert resolution in backtesting, live can use other data source
// for example daily data for options
if (!_liveMode)
{
var supportedResolutions = baseInstance.SupportedResolutions();
if (supportedResolutions.Contains(resolution.Value))
{
continue;
}
throw new ArgumentException($"Sorry {resolution.ToStringInvariant()} is not a supported resolution for {typeTuple.Item1.Name}" +
$" and SecurityType.{symbol.SecurityType.ToStringInvariant()}." +
$" Please change your AddData to use one of the supported resolutions ({string.Join(",", supportedResolutions)}).");
}
}
}
MarketHoursDatabase.Entry marketHoursDbEntry;
if (!_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType, out marketHoursDbEntry))
{
if (symbol.SecurityType == SecurityType.Base)
{
var baseInstance = dataTypes.Single().Item1.GetBaseDataInstance();
baseInstance.Symbol = symbol;
_marketHoursDatabase.SetEntryAlwaysOpen(symbol.ID.Market, null, SecurityType.Base, baseInstance.DataTimeZone());
}
marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
}
var exchangeHours = marketHoursDbEntry.ExchangeHours;
if (symbol.ID.SecurityType.IsOption() ||
symbol.ID.SecurityType == SecurityType.Future ||
symbol.ID.SecurityType == SecurityType.Index)
{
dataNormalizationMode = DataNormalizationMode.Raw;
}
if (marketHoursDbEntry.DataTimeZone == null)
{
throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
"DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
}
if (exchangeHours.TimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
"ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
}
var result = (from subscriptionDataType in dataTypes
let dataType = subscriptionDataType.Item1
let tickType = subscriptionDataType.Item2
select new SubscriptionDataConfig(
dataType,
symbol,
resolution.Value,
marketHoursDbEntry.DataTimeZone,
exchangeHours.TimeZone,
fillForward,
extendedMarketHours,
// if the subscription data types were not provided and the tick type is OpenInterest we make it internal
subscriptionDataTypes == null && tickType == TickType.OpenInterest || isInternalFeed,
isCustomData,
isFilteredSubscription: isFilteredSubscription,
tickType: tickType,
dataNormalizationMode: dataNormalizationMode)).ToList();
for (int i = 0; i < result.Count; i++)
{
result[i] = SubscriptionManagerGetOrAdd(result[i]);
// track all registered data types
_registeredTypesProvider.RegisterType(result[i].Type);
}
return(result);
}
