private Security GetSecurity(Symbol symbol, DataNormalizationMode mode)
{
var symbolProperties = SymbolPropertiesDatabase.FromDataFolder()
.GetSymbolProperties(symbol.ID.Market, symbol.Value, symbol.ID.SecurityType, CashBook.AccountCurrency);
Security security;
if (symbol.ID.SecurityType == SecurityType.Equity)
{
security = new QuantConnect.Securities.Equity.Equity(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork, true, true, false),
new Cash(CashBook.AccountCurrency, 0, 1m),
symbolProperties);
}
else
{
security = new QuantConnect.Securities.Forex.Forex(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
new Cash(CashBook.AccountCurrency, 0, 1m),
new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork, true, true, false),
symbolProperties);
}
var TimeKeeper = new TimeKeeper(DateTime.Now.ConvertToUtc(TimeZones.NewYork), new[] { TimeZones.NewYork });
security.SetLocalTimeKeeper(TimeKeeper.GetLocalTimeKeeper(TimeZones.NewYork));
security.SetDataNormalizationMode(mode);
return security;
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public override void SetDataNormalizationMode(DataNormalizationMode mode)
{
base.SetDataNormalizationMode(mode);
if (mode == DataNormalizationMode.Adjusted)
{
PriceVariationModel = new AdjustedPriceVariationModel();
}
else
{
PriceVariationModel = new EquityPriceVariationModel();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="HistoryRequest"/> class from the specified config and exchange hours
/// </summary>
/// <param name="config">The subscription data config used to initalize this request</param>
/// <param name="hours">The exchange hours used for fill forward processing</param>
/// <param name="startTimeUtc">The start time for this request,</param>
/// <param name="endTimeUtc">The start time for this request</param>
public HistoryRequest(SubscriptionDataConfig config, SecurityExchangeHours hours, DateTime startTimeUtc, DateTime endTimeUtc)
{
StartTimeUtc = startTimeUtc;
EndTimeUtc = endTimeUtc;
Symbol = config.Symbol;
ExchangeHours = hours;
Resolution = config.Resolution;
FillForwardResolution = config.FillDataForward ? config.Resolution : (Resolution?)null;
IncludeExtendedMarketHours = config.ExtendedMarketHours;
DataType = config.Type;
IsCustomData = config.IsCustomData;
DataNormalizationMode = config.DataNormalizationMode;
TimeZone = config.DataTimeZone;
}
public void HistoryDoesNotThrowForSupportedDataNormalizationMode_Equity(DataNormalizationMode dataNormalizationMode)
{
_algorithm = GetAlgorithmWithEquity(new DateTime(2014, 6, 6));
Assert.AreEqual(2, _algorithm.SubscriptionManager.Subscriptions.ToList().Count);
var equity = _algorithm.SubscriptionManager.Subscriptions.First();
Assert.AreEqual(SecurityType.Equity, equity.SecurityType);
Assert.DoesNotThrow(() =>
{
_algorithm.History(new [] { equity.Symbol }, _algorithm.Time.AddDays(-1), _algorithm.Time, equity.Resolution,
dataNormalizationMode: dataNormalizationMode).ToList();
});
}
public void HistoryDoesNotThrowForSupportedDataNormalizationMode_Future(DataNormalizationMode dataNormalizationMode)
{
_algorithm = GetAlgorithmWithFuture(new DateTime(2014, 1, 1));
Assert.IsNotEmpty(_algorithm.SubscriptionManager.Subscriptions);
var future = _algorithm.SubscriptionManager.Subscriptions.First();
Assert.AreEqual(SecurityType.Future, future.SecurityType);
Assert.DoesNotThrow(() =>
{
_algorithm.History(new [] { future.Symbol }, _algorithm.StartDate, _algorithm.Time, future.Resolution,
dataNormalizationMode: dataNormalizationMode).ToList();
});
}
public void SubscriptionHistoryRequestWithDifferentDataNormalizationModes_Equity()
{
var dataNormalizationModes = new DataNormalizationMode[] {
DataNormalizationMode.Raw,
DataNormalizationMode.Adjusted,
DataNormalizationMode.SplitAdjusted
};
_algorithm = GetAlgorithmWithEquity(new DateTime(2014, 6, 6));
var equity = _algorithm.SubscriptionManager.Subscriptions.First();
CheckHistoryResultsForDataNormalizationModes(_algorithm, equity.Symbol, _algorithm.Time.AddDays(-1), _algorithm.Time, equity.Resolution,
dataNormalizationModes);
}
public void SubscriptionHistoryRequestWithDifferentDataNormalizationModes_Future()
{
var dataNormalizationModes = new DataNormalizationMode[] {
DataNormalizationMode.Raw,
DataNormalizationMode.BackwardsRatio,
DataNormalizationMode.BackwardsPanamaCanal,
DataNormalizationMode.ForwardPanamaCanal
};
_algorithm = GetAlgorithmWithFuture(new DateTime(2014, 1, 1));
var future = _algorithm.SubscriptionManager.Subscriptions.First();
CheckHistoryResultsForDataNormalizationModes(_algorithm, future.Symbol, new DateTime(2013, 10, 6), _algorithm.Time, future.Resolution,
dataNormalizationModes);
}
/// <summary>
/// Initializes a new default instance of the <see cref="HistoryRequest"/> class
/// </summary>
public HistoryRequest()
{
StartTimeUtc = EndTimeUtc = DateTime.UtcNow;
Symbol = Symbol.Empty;
ExchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
Resolution = Resolution.Minute;
FillForwardResolution = Resolution.Minute;
IncludeExtendedMarketHours = false;
DataType = typeof(TradeBar);
SecurityType = SecurityType.Equity;
TimeZone = TimeZones.NewYork;
Market = QuantConnect.Market.USA;
IsCustomData = false;
DataNormalizationMode = DataNormalizationMode.Adjusted;
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data type to use.
/// If the config already existed will return existing instance instead
/// </summary>
public SubscriptionDataConfig Add(
Type dataType,
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
)
{
return(Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))
}, dataNormalizationMode)
.First());
}
private static decimal GetScaleFactor(FactorFile factorFile, DataNormalizationMode mode, DateTime date)
{
switch (mode)
{
case DataNormalizationMode.Raw:
return(1);
case DataNormalizationMode.TotalReturn:
case DataNormalizationMode.SplitAdjusted:
return(factorFile.GetSplitFactor(date));
case DataNormalizationMode.Adjusted:
return(factorFile.GetPriceScaleFactor(date));
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Resolves the price scale for a date given a factor file and required settings
/// </summary>
/// <param name="factorFile">The factor file to use</param>
/// <param name="dateTime">The date for the price scale lookup</param>
/// <param name="normalizationMode">The price normalization mode requested</param>
/// <param name="contractOffset">The contract offset, useful for continuous contracts</param>
/// <param name="dataMappingMode">The data mapping mode used, useful for continuous contracts</param>
/// <returns>The price scale to use</returns>
public static decimal GetPriceScale(
this IFactorProvider factorFile,
DateTime dateTime,
DataNormalizationMode normalizationMode,
uint contractOffset = 0,
DataMappingMode?dataMappingMode = null
)
{
if (factorFile == null)
{
if (normalizationMode is DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
{
return(0);
}
return(1);
}
return(factorFile.GetPriceFactor(dateTime, normalizationMode, dataMappingMode, contractOffset));
}
public override void OnEndOfAlgorithm()
{
var equityDataNormalizationModes = new DataNormalizationMode[] {
DataNormalizationMode.Raw,
DataNormalizationMode.Adjusted,
DataNormalizationMode.SplitAdjusted
};
CheckHistoryResultsForDataNormalizationModes(_aaplEquitySymbol, StartDate, EndDate, Resolution.Daily, equityDataNormalizationModes);
var futureDataNormalizationModes = new DataNormalizationMode[] {
DataNormalizationMode.Raw,
DataNormalizationMode.BackwardsRatio,
DataNormalizationMode.BackwardsPanamaCanal,
DataNormalizationMode.ForwardPanamaCanal
};
CheckHistoryResultsForDataNormalizationModes(_esFutureSymbol, StartDate, EndDate, Resolution.Daily, futureDataNormalizationModes);
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public void SetDataNormalizationMode(DataNormalizationMode mode)
{
foreach (var subscription in SubscriptionsBag)
{
subscription.DataNormalizationMode = mode;
}
if (Type == SecurityType.Equity)
{
if (mode == DataNormalizationMode.Adjusted)
{
PriceVariationModel = new AdjustedPriceVariationModel();
}
else
{
PriceVariationModel = new EquityPriceVariationModel();
}
}
}
/// <summary>
/// Applies a dividend to the portfolio
/// </summary>
/// <param name="dividend">The dividend to be applied</param>
/// <param name="liveMode">True if live mode, false for backtest</param>
/// <param name="mode">The <see cref="DataNormalizationMode"/> for this security</param>
public void ApplyDividend(Dividend dividend, bool liveMode, DataNormalizationMode mode)
{
// we currently don't properly model dividend payable dates, so in
// live mode it's more accurate to rely on the brokerage cash sync
if (liveMode)
{
return;
}
var security = Securities[dividend.Symbol];
// only apply dividends when we're in raw mode or split adjusted mode
if (mode == DataNormalizationMode.Raw || mode == DataNormalizationMode.SplitAdjusted)
{
// longs get benefits, shorts get clubbed on dividends
var total = security.Holdings.Quantity * dividend.Distribution;
// assuming USD, we still need to add Currency to the security object
_baseCurrencyCash.AddAmount(total);
}
}
public void CheckSecurityMinimumPriceVariation(string ticker, SecurityType securityType, string market, DataNormalizationMode mode)
{
var symbol = Symbol.Create(ticker, securityType, market);
var security = GetSecurity(symbol, mode);
var expected = security.SymbolProperties.MinimumPriceVariation;
var adjutedEquity = mode == DataNormalizationMode.Adjusted && securityType == SecurityType.Equity;
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, 10m));
var actual = security.PriceVariationModel.GetMinimumPriceVariation(security);
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, 1m));
actual = security.PriceVariationModel.GetMinimumPriceVariation(security);
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
// Special case, if stock price less than $1, minimum price variation is $0.0001
if (securityType == SecurityType.Equity) expected = 0.0001m;
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, .99m));
actual = security.PriceVariationModel.GetMinimumPriceVariation(security);
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
}
/// <summary>
/// Add Market Data Required - generic data typing support as long as Type implements BaseData.
/// </summary>
/// <param name="dataType">Set the type of the data we're subscribing to.</param>
/// <param name="tickType">Tick type for the subscription.</param>
/// <param name="symbol">Symbol of the asset we're like</param>
/// <param name="resolution">Resolution of Asset Required</param>
/// <param name="dataTimeZone">The time zone the subscription's data is time stamped in</param>
/// <param name="exchangeTimeZone">
/// Specifies the time zone of the exchange for the security this subscription is for. This
/// is this output time zone, that is, the time zone that will be used on BaseData instances
/// </param>
/// <param name="isCustomData">True if this is custom user supplied data, false for normal QC data</param>
/// <param name="fillDataForward">when there is no data pass the last tradebar forward</param>
/// <param name="extendedMarketHours">Request premarket data as well when true </param>
/// <param name="isInternalFeed">
/// Set to true to prevent data from this subscription from being sent into the algorithm's
/// OnData events
/// </param>
/// <param name="isFilteredSubscription">
/// True if this subscription should have filters applied to it (market hours/user
/// filters from security), false otherwise
/// </param>
/// <param name="dataNormalizationMode">Define how data is normalized</param>
/// <returns>
/// The newly created <see cref="SubscriptionDataConfig" /> or existing instance if it already existed
/// </returns>
public SubscriptionDataConfig Add(
Type dataType,
TickType tickType,
Symbol symbol,
Resolution resolution,
DateTimeZone dataTimeZone,
DateTimeZone exchangeTimeZone,
bool isCustomData,
bool fillDataForward = true,
bool extendedMarketHours = false,
bool isInternalFeed = false,
bool isFilteredSubscription = true,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
)
{
return(SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward,
extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(dataType, tickType)
},
dataNormalizationMode).First());
}
/// <summary>
/// Initializes a new instance of the <see cref="HistoryRequest"/> class from the specified parameters
/// </summary>
/// <param name="startTimeUtc">The start time for this request,</param>
/// <param name="endTimeUtc">The start time for this request</param>
/// <param name="dataType">The data type of the output data</param>
/// <param name="symbol">The symbol to request data for</param>
/// <param name="resolution">The requested data resolution</param>
/// <param name="exchangeHours">The exchange hours used in fill forward processing</param>
/// <param name="fillForwardResolution">The requested fill forward resolution for this request</param>
/// <param name="includeExtendedMarketHours">True to include data from pre/post market hours</param>
/// <param name="isCustomData">True for custom user data, false for normal QC data</param>
/// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
public HistoryRequest(DateTime startTimeUtc,
DateTime endTimeUtc,
Type dataType,
Symbol symbol,
Resolution resolution,
SecurityExchangeHours exchangeHours,
Resolution?fillForwardResolution,
bool includeExtendedMarketHours,
bool isCustomData,
DataNormalizationMode dataNormalizationMode
)
{
StartTimeUtc = startTimeUtc;
EndTimeUtc = endTimeUtc;
Symbol = symbol;
ExchangeHours = exchangeHours;
Resolution = resolution;
FillForwardResolution = fillForwardResolution;
IncludeExtendedMarketHours = includeExtendedMarketHours;
DataType = dataType;
IsCustomData = isCustomData;
DataNormalizationMode = dataNormalizationMode;
TimeZone = exchangeHours.TimeZone;
}
public void CheckUniverseSelectionSecurityDataNormalizationMode(DataNormalizationMode dataNormalizationMode)
{
var tuple = GetAlgorithmAndDataManager();
var algorithm = tuple.Item1;
var dataManager = tuple.Item2;
var symbol = Symbols.SPY;
algorithm.UniverseSettings.DataNormalizationMode = dataNormalizationMode;
algorithm.AddUniverse(coarse => new[] { symbol });
var changes = dataManager.UniverseSelection
.ApplyUniverseSelection(
algorithm.UniverseManager.First().Value,
algorithm.UtcTime,
new BaseDataCollection(algorithm.UtcTime, null, Enumerable.Empty <CoarseFundamental>()));
Assert.AreEqual(1, changes.AddedSecurities.Count());
var security = changes.AddedSecurities.First();
Assert.AreEqual(symbol, security.Symbol);
Assert.AreEqual(dataNormalizationMode, security.DataNormalizationMode);
}
/// <summary>
/// Gets the price scale factor that includes dividend and split adjustments for the specified search date
/// </summary>
public override decimal GetPriceFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode?dataMappingMode = null, uint contractOffset = 0)
{
if (dataNormalizationMode == DataNormalizationMode.Raw)
{
return(0);
}
var factor = 1m;
for (var i = 0; i < _reversedFactorFileDates.Count; i++)
{
var factorDate = _reversedFactorFileDates[i];
if (factorDate.Date < searchDate.Date)
{
break;
}
var factorFileRow = SortedFactorFileData[factorDate];
switch (dataNormalizationMode)
{
case DataNormalizationMode.TotalReturn:
case DataNormalizationMode.SplitAdjusted:
factor = factorFileRow.First().SplitFactor;
break;
case DataNormalizationMode.Adjusted:
factor = factorFileRow.First().PriceScaleFactor;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
return(factor);
}
/// <summary>
/// Initializes a new instance of the <see cref="HistoryRequest"/> class from the specified parameters
/// </summary>
/// <param name="startTimeUtc">The start time for this request,</param>
/// <param name="endTimeUtc">The start time for this request</param>
/// <param name="dataType">The data type of the output data</param>
/// <param name="symbol">The symbol to request data for</param>
/// <param name="resolution">The requested data resolution</param>
/// <param name="exchangeHours">The exchange hours used in fill forward processing</param>
/// <param name="dataTimeZone">The time zone of the data</param>
/// <param name="fillForwardResolution">The requested fill forward resolution for this request</param>
/// <param name="includeExtendedMarketHours">True to include data from pre/post market hours</param>
/// <param name="isCustomData">True for custom user data, false for normal QC data</param>
/// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
/// <param name="tickType">The tick type used to created the <see cref="SubscriptionDataConfig"/> for the retrieval of history data</param>
public HistoryRequest(DateTime startTimeUtc,
DateTime endTimeUtc,
Type dataType,
Symbol symbol,
Resolution resolution,
SecurityExchangeHours exchangeHours,
DateTimeZone dataTimeZone,
Resolution?fillForwardResolution,
bool includeExtendedMarketHours,
bool isCustomData,
DataNormalizationMode dataNormalizationMode,
TickType tickType)
: base(startTimeUtc, endTimeUtc, exchangeHours, tickType)
{
Symbol = symbol;
DataTimeZone = dataTimeZone;
Resolution = resolution;
FillForwardResolution = fillForwardResolution;
IncludeExtendedMarketHours = includeExtendedMarketHours;
DataType = dataType;
IsCustomData = isCustomData;
DataNormalizationMode = dataNormalizationMode;
TickType = tickType;
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data types to use.
/// If the config already existed will return existing instance instead
/// </summary>
public List <SubscriptionDataConfig> Add(
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
List <Tuple <Type, TickType> > subscriptionDataTypes = null,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
)
{
var dataTypes = subscriptionDataTypes ??
LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution ?? Resolution.Minute, symbol.IsCanonical());
if (!dataTypes.Any())
{
throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
}
var resolutionWasProvided = resolution.HasValue;
foreach (var typeTuple in dataTypes)
{
var baseInstance = typeTuple.Item1.GetBaseDataInstance();
baseInstance.Symbol = symbol;
if (!resolutionWasProvided)
{
var defaultResolution = baseInstance.DefaultResolution();
if (resolution.HasValue && resolution != defaultResolution)
{
// we are here because there are multiple 'dataTypes'.
// if we get different default resolutions lets throw, this shouldn't happen
throw new InvalidOperationException(
$"Different data types ({string.Join(",", dataTypes.Select(tuple => tuple.Item1))})" +
$" provided different default resolutions {defaultResolution} and {resolution}, this is an unexpected invalid operation.");
}
resolution = defaultResolution;
}
else
{
// only assert resolution in backtesting, live can use other data source
// for example daily data for options
if (!_liveMode)
{
var supportedResolutions = baseInstance.SupportedResolutions();
if (supportedResolutions.Contains(resolution.Value))
{
continue;
}
throw new ArgumentException($"Sorry {resolution.ToStringInvariant()} is not a supported resolution for {typeTuple.Item1.Name}" +
$" and SecurityType.{symbol.SecurityType.ToStringInvariant()}." +
$" Please change your AddData to use one of the supported resolutions ({string.Join(",", supportedResolutions)}).");
}
}
}
MarketHoursDatabase.Entry marketHoursDbEntry;
if (!_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType, out marketHoursDbEntry))
{
if (symbol.SecurityType == SecurityType.Base)
{
var baseInstance = dataTypes.Single().Item1.GetBaseDataInstance();
baseInstance.Symbol = symbol;
_marketHoursDatabase.SetEntryAlwaysOpen(symbol.ID.Market, null, SecurityType.Base, baseInstance.DataTimeZone());
}
marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
}
var exchangeHours = marketHoursDbEntry.ExchangeHours;
if (symbol.ID.SecurityType.IsOption() ||
symbol.ID.SecurityType == SecurityType.Future ||
symbol.ID.SecurityType == SecurityType.Index)
{
dataNormalizationMode = DataNormalizationMode.Raw;
}
if (marketHoursDbEntry.DataTimeZone == null)
{
throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
"DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
}
if (exchangeHours.TimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
"ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
}
var result = (from subscriptionDataType in dataTypes
let dataType = subscriptionDataType.Item1
let tickType = subscriptionDataType.Item2
select new SubscriptionDataConfig(
dataType,
symbol,
resolution.Value,
marketHoursDbEntry.DataTimeZone,
exchangeHours.TimeZone,
fillForward,
extendedMarketHours,
// if the subscription data types were not provided and the tick type is OpenInterest we make it internal
subscriptionDataTypes == null && tickType == TickType.OpenInterest || isInternalFeed,
isCustomData,
isFilteredSubscription: isFilteredSubscription,
tickType: tickType,
dataNormalizationMode: dataNormalizationMode)).ToList();
for (int i = 0; i < result.Count; i++)
{
result[i] = SubscriptionManagerGetOrAdd(result[i]);
// track all registered data types
_registeredTypesProvider.RegisterType(result[i].Type);
}
return(result);
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public void SetDataNormalizationMode(DataNormalizationMode mode)
{
foreach (var subscription in SubscriptionsBag)
{
subscription.DataNormalizationMode = mode;
}
if (Type == SecurityType.Equity)
{
if (mode == DataNormalizationMode.Adjusted)
{
PriceVariationModel = new AdjustedPriceVariationModel();
}
else
{
PriceVariationModel = new EquityPriceVariationModel();
}
}
}
/// <summary>
/// Applies a split to the portfolio
/// </summary>
/// <param name="split">The split to be applied</param>
/// <param name="liveMode">True if live mode, false for backtest</param>
/// <param name="mode">The <see cref="DataNormalizationMode"/> for this security</param>
public void ApplySplit(Split split, bool liveMode, DataNormalizationMode mode)
{
var security = Securities[split.Symbol];
// only apply splits to equities
if (security.Type != SecurityType.Equity)
{
return;
}
// only apply splits in live or raw data mode
if (!liveMode && mode != DataNormalizationMode.Raw)
{
return;
}
// we need to modify our holdings in lght of the split factor
var quantity = security.Holdings.Quantity / split.SplitFactor;
var avgPrice = security.Holdings.AveragePrice * split.SplitFactor;
// we'll model this as a cash adjustment
var leftOver = quantity - (int)quantity;
var extraCash = leftOver * split.ReferencePrice;
_baseCurrencyCash.AddAmount(extraCash);
security.Holdings.SetHoldings(avgPrice, (int)quantity);
// build a 'next' value to update the market prices in light of the split factor
var next = security.GetLastData();
if (next == null)
{
// sometimes we can get splits before we receive data which
// will cause this to return null, in this case we can't possibly
// have any holdings or price to set since we haven't received
// data yet, so just do nothing
return;
}
next.Value *= split.SplitFactor;
// make sure to modify open/high/low as well for tradebar data types
var tradeBar = next as TradeBar;
if (tradeBar != null)
{
tradeBar.Open *= split.SplitFactor;
tradeBar.High *= split.SplitFactor;
tradeBar.Low *= split.SplitFactor;
}
// make sure to modify bid/ask as well for tradebar data types
var tick = next as Tick;
if (tick != null)
{
tick.AskPrice *= split.SplitFactor;
tick.BidPrice *= split.SplitFactor;
}
security.SetMarketPrice(next);
// security price updated
InvalidateTotalPortfolioValue();
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public void SetDataNormalizationMode(DataNormalizationMode mode)
{
foreach (var subscription in SubscriptionsBag)
{
subscription.DataNormalizationMode = mode;
}
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public void SetDataNormalizationMode(DataNormalizationMode mode)
{
_config.DataNormalizationMode = mode;
}
public void CheckSecurityMinimumPriceVariation(string ticker, SecurityType securityType, string market, DataNormalizationMode mode)
{
var symbol = Symbol.Create(ticker, securityType, market);
var security = GetSecurity(symbol, mode);
var expected = security.SymbolProperties.MinimumPriceVariation;
var adjutedEquity = mode == DataNormalizationMode.Adjusted && securityType == SecurityType.Equity;
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, 10m));
var actual = security.PriceVariationModel.GetMinimumPriceVariation(
new GetMinimumPriceVariationParameters(security, security.Price));
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, 1m));
actual = security.PriceVariationModel.GetMinimumPriceVariation(
new GetMinimumPriceVariationParameters(security, security.Price));
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
// Special case, if stock price less than $1, minimum price variation is $0.0001
if (securityType == SecurityType.Equity)
{
expected = 0.0001m;
}
security.SetMarketPrice(new IndicatorDataPoint(symbol, DateTime.Now, .99m));
actual = security.PriceVariationModel.GetMinimumPriceVariation(
new GetMinimumPriceVariationParameters(security, security.Price));
Assert.AreEqual(adjutedEquity ? 0 : expected, actual);
}
/// <summary>
/// Initializes a new instance of the <see cref="CustomSecurityInitializer"/> class
/// with the specified normalization mode
/// </summary>
/// <param name="brokerageModel">The brokerage model used to get fill/fee/slippage/settlement models</param>
/// <param name="dataNormalizationMode">The desired data normalization mode</param>
public CustomSecurityInitializer(IBrokerageModel brokerageModel, DataNormalizationMode dataNormalizationMode)
: base(brokerageModel)
{
_dataNormalizationMode = dataNormalizationMode;
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data types to use.
/// If the config already existed will return existing instance instead
/// </summary>
public List <SubscriptionDataConfig> Add(
Symbol symbol,
Resolution resolution,
bool fillForward,
bool extendedMarketHours,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
List <Tuple <Type, TickType> > subscriptionDataTypes = null,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
)
{
var dataTypes = subscriptionDataTypes ??
LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution, symbol.IsCanonical());
var marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType);
var exchangeHours = marketHoursDbEntry.ExchangeHours;
if (symbol.ID.SecurityType == SecurityType.Option || symbol.ID.SecurityType == SecurityType.Future)
{
dataNormalizationMode = DataNormalizationMode.Raw;
}
if (marketHoursDbEntry.DataTimeZone == null)
{
throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
"DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
}
if (exchangeHours.TimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
"ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
}
if (!dataTypes.Any())
{
throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
}
var result = (from subscriptionDataType in dataTypes
let dataType = subscriptionDataType.Item1
let tickType = subscriptionDataType.Item2
select new SubscriptionDataConfig(
dataType,
symbol,
resolution,
marketHoursDbEntry.DataTimeZone,
exchangeHours.TimeZone,
fillForward,
extendedMarketHours,
isInternalFeed,
isCustomData,
isFilteredSubscription: isFilteredSubscription,
tickType: tickType,
dataNormalizationMode: dataNormalizationMode)).ToList();
for (int i = 0; i < result.Count; i++)
{
result[i] = SubscriptionManagerGetOrAdd(result[i]);
// track all registered data types
_registeredTypesProvider.RegisterType(result[i].Type);
}
return(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="CustomSecurityInitializer"/> class
/// with the specified normalization mode
/// </summary>
/// <param name="brokerageModel">The brokerage model used to get fill/fee/slippage/settlement models</param>
/// <param name="dataNormalizationMode">The desired data normalization mode</param>
public CustomSecurityInitializer(IBrokerageModel brokerageModel, DataNormalizationMode dataNormalizationMode)
: base(brokerageModel)
{
_dataNormalizationMode = dataNormalizationMode;
}
/// <summary>
/// Sets the data normalization mode to be used by this security
/// </summary>
public void SetDataNormalizationMode(DataNormalizationMode mode)
{
_config.DataNormalizationMode = mode;
}
/// <summary>
/// Initializes a new instance of the <see cref="UniverseSettings"/> class
/// </summary>
/// <param name="resolution">The resolution</param>
/// <param name="leverage">The leverage to be used</param>
/// <param name="fillForward">True to fill data forward, false otherwise</param>
/// <param name="extendedMarketHours">True to allow extended market hours data, false otherwise</param>
/// <param name="minimumTimeInUniverse">Defines the minimum amount of time a security must remain in the universe before being removed</param>
/// <param name="dataNormalizationMode">Defines how universe data is normalized before being send into the algorithm</param>
public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
{
Resolution = resolution;
Leverage = leverage;
FillForward = fillForward;
ExtendedMarketHours = extendedMarketHours;
MinimumTimeInUniverse = minimumTimeInUniverse;
DataNormalizationMode = dataNormalizationMode;
}
/// <summary>
/// Clones the data, computes the utc emit time and performs exchange round down behavior, storing the result in a new <see cref="SubscriptionData"/> instance
/// </summary>
/// <param name="configuration">The subscription's configuration</param>
/// <param name="exchangeHours">The exchange hours of the security</param>
/// <param name="offsetProvider">The subscription's offset provider</param>
/// <param name="data">The data being emitted</param>
/// <param name="normalizationMode">Specifies how data is normalized</param>
/// <param name="factor">price scale factor</param>
/// <returns>A new <see cref="SubscriptionData"/> containing the specified data</returns>
public static SubscriptionData Create(SubscriptionDataConfig configuration, SecurityExchangeHours exchangeHours, TimeZoneOffsetProvider offsetProvider, BaseData data, DataNormalizationMode normalizationMode, decimal?factor = null)
{
if (data == null)
{
return(null);
}
data = data.Clone(data.IsFillForward);
var emitTimeUtc = offsetProvider.ConvertToUtc(data.EndTime);
// Let's round down for any data source that implements a time delta between
// the start of the data and end of the data (usually used with Bars).
// The time delta ensures that the time collected from `EndTime` has
// no look-ahead bias, and is point-in-time.
if (data.Time != data.EndTime)
{
data.Time = data.Time.ExchangeRoundDownInTimeZone(configuration.Increment, exchangeHours, configuration.DataTimeZone, configuration.ExtendedMarketHours);
}
if (factor.HasValue && (factor.Value != 1 || configuration.SumOfDividends != 0))
{
var sumOfDividends = configuration.SumOfDividends;
var normalizedData = data.Clone(data.IsFillForward);
if (normalizationMode == DataNormalizationMode.Adjusted || normalizationMode == DataNormalizationMode.SplitAdjusted)
{
normalizedData.Adjust(factor.Value);
}
else if (normalizationMode == DataNormalizationMode.TotalReturn)
{
normalizedData.Scale(p => p * factor.Value + sumOfDividends, 1 / factor.Value);
}
return(new PrecalculatedSubscriptionData(configuration, data, normalizedData, normalizationMode, emitTimeUtc));
}
return(new SubscriptionData(data, emitTimeUtc));
}
/// <summary>
/// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
/// </summary>
/// <param name="pyObject">The symbols to retrieve historical data for</param>
/// <param name="start">The start time in the algorithm's time zone</param>
/// <param name="end">The end time in the algorithm's time zone</param>
/// <param name="resolution">The resolution to request</param>
/// <param name="selector">Selects a value from the BaseData to filter the request output, if null retuns all OHLCV</param>
/// <param name="dataNormalizationMode">The data normalization mode. Default is Adjusted</param>
/// <param name="extendedMarket">True to include extended market hours data, false otherwise</param>
/// <returns>A pandas.DataFrame containing the requested historical data</returns>
public PyObject History(PyObject pyObject, DateTime?start = null, DateTime?end = null, Resolution resolution = Resolution.Daily, Func <IBaseData, decimal> selector = null, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted, bool extendedMarket = false)
{
var symbols = GetSymbolsFromPyObject(pyObject);
if (symbols == null)
{
return("Invalid ticker(s). Please use get_symbol to add symbols.".ToPython());
}
var requests = symbols.Select(symbol =>
{
if (symbol.SecurityType == SecurityType.Forex || symbol.SecurityType == SecurityType.Cfd)
{
start = start ?? new DateTime(2005, 1, 1);
}
return(new HistoryRequest()
{
Symbol = symbol,
StartTimeUtc = start ?? symbol.ID.Date,
EndTimeUtc = end ?? DateTime.Now,
Resolution = resolution,
FillForwardResolution = null,
DataNormalizationMode = dataNormalizationMode,
IncludeExtendedMarketHours = extendedMarket,
DataType = symbol.ID.SecurityType == SecurityType.Equity ? typeof(TradeBar) : typeof(QuoteBar)
});
});
return(CreateDataFrame(requests, _historyProvider.GetHistory(requests, TimeZones.NewYork), selector));
}
/// <summary>
/// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
/// </summary>
/// <param name="symbols">The symbols to retrieve historical data for</param>
/// <param name="span">The span over which to retrieve recent historical data</param>
/// <param name="resolution">The resolution to request</param>
/// <param name="selector">Selects a value from the BaseData to filter the request output, if null retuns all OHLCV</param>
/// <param name="dataNormalizationMode">The data normalization mode. Default is Adjusted</param>
/// <param name="extendedMarket">True to include extended market hours data, false otherwise</param>
/// <returns>A pandas.DataFrame containing the requested historical data</returns>
public PyObject History(PyObject symbols, TimeSpan span, Resolution resolution = Resolution.Daily, Func <IBaseData, decimal> selector = null, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted, bool extendedMarket = false)
{
var endTimeUtc = DateTime.Now;
var startTimeUtc = endTimeUtc - span;
return(History(symbols, startTimeUtc, endTimeUtc, resolution, selector, dataNormalizationMode, extendedMarket));
}
/// <summary>
/// Constructor for Data Subscriptions
/// </summary>
/// <param name="objectType">Type of the data objects.</param>
/// <param name="symbol">Symbol of the asset we're requesting</param>
/// <param name="resolution">Resolution of the asset we're requesting</param>
/// <param name="dataTimeZone">The time zone the raw data is time stamped in</param>
/// <param name="exchangeTimeZone">Specifies the time zone of the exchange for the security this subscription is for. This
/// is this output time zone, that is, the time zone that will be used on BaseData instances</param>
/// <param name="fillForward">Fill in gaps with historical data</param>
/// <param name="extendedHours">Equities only - send in data from 4am - 8pm</param>
/// <param name="isInternalFeed">Set to true if this subscription is added for the sole purpose of providing currency conversion rates,
/// setting this flag to true will prevent the data from being sent into the algorithm's OnData methods</param>
/// <param name="isCustom">True if this is user supplied custom data, false for normal QC data</param>
/// <param name="tickType">Specifies if trade or quote data is subscribed</param>
/// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
/// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
public SubscriptionDataConfig(Type objectType,
Symbol symbol,
Resolution resolution,
DateTimeZone dataTimeZone,
DateTimeZone exchangeTimeZone,
bool fillForward,
bool extendedHours,
bool isInternalFeed,
bool isCustom = false,
TickType?tickType = null,
bool isFilteredSubscription = true,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
{
if (objectType == null)
{
throw new ArgumentNullException("objectType");
}
if (symbol == null)
{
throw new ArgumentNullException("symbol");
}
if (dataTimeZone == null)
{
throw new ArgumentNullException("dataTimeZone");
}
if (exchangeTimeZone == null)
{
throw new ArgumentNullException("exchangeTimeZone");
}
Type = objectType;
SecurityType = symbol.ID.SecurityType;
Resolution = resolution;
_sid = symbol.ID;
Symbol = symbol;
FillDataForward = fillForward;
ExtendedMarketHours = extendedHours;
PriceScaleFactor = 1;
IsInternalFeed = isInternalFeed;
IsCustomData = isCustom;
Market = symbol.ID.Market;
DataTimeZone = dataTimeZone;
ExchangeTimeZone = exchangeTimeZone;
IsFilteredSubscription = isFilteredSubscription;
Consolidators = new HashSet <IDataConsolidator>();
DataNormalizationMode = dataNormalizationMode;
TickType = tickType ?? LeanData.GetCommonTickTypeForCommonDataTypes(objectType, SecurityType);
switch (resolution)
{
case Resolution.Tick:
//Ticks are individual sales and fillforward doesn't apply.
Increment = TimeSpan.FromSeconds(0);
FillDataForward = false;
break;
case Resolution.Second:
Increment = TimeSpan.FromSeconds(1);
break;
case Resolution.Minute:
Increment = TimeSpan.FromMinutes(1);
break;
case Resolution.Hour:
Increment = TimeSpan.FromHours(1);
break;
case Resolution.Daily:
Increment = TimeSpan.FromDays(1);
break;
default:
throw new InvalidEnumArgumentException("Unexpected Resolution: " + resolution);
}
}
/// <summary>
/// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
/// </summary>
/// <param name="symbols">The symbols to retrieve historical data for</param>
/// <param name="periods">The number of bars to request</param>
/// <param name="resolution">The resolution to request</param>
/// <param name="selector">Selects a value from the BaseData to filter the request output, if null retuns all OHLCV</param>
/// <param name="dataNormalizationMode">The data normalization mode. Default is Adjusted</param>
/// <param name="extendedMarket">True to include extended market hours data, false otherwise</param>
/// <returns>A pandas.DataFrame containing the requested historical data</returns>
public PyObject History(PyObject symbols, int periods, Resolution resolution = Resolution.Daily, Func <IBaseData, decimal> selector = null, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted, bool extendedMarket = false)
{
var span = TimeSpan.FromDays(periods);
switch (resolution)
{
case Resolution.Second:
span = TimeSpan.FromSeconds(periods);
break;
case Resolution.Minute:
span = TimeSpan.FromMinutes(periods);
break;
case Resolution.Hour:
span = TimeSpan.FromHours(periods);
break;
default:
break;
}
return(History(symbols, span, resolution, selector, dataNormalizationMode, extendedMarket));
}