/// <summary>
/// Initializes a new instance of the <see cref="HistoryRequest"/> class from the specified parameters
/// </summary>
/// <param name="startTimeUtc">The start time for this request,</param>
/// <param name="endTimeUtc">The start time for this request</param>
/// <param name="dataType">The data type of the output data</param>
/// <param name="symbol">The symbol to request data for</param>
/// <param name="resolution">The requested data resolution</param>
/// <param name="exchangeHours">The exchange hours used in fill forward processing</param>
/// <param name="dataTimeZone">The time zone of the data</param>
/// <param name="fillForwardResolution">The requested fill forward resolution for this request</param>
/// <param name="includeExtendedMarketHours">True to include data from pre/post market hours</param>
/// <param name="isCustomData">True for custom user data, false for normal QC data</param>
/// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
/// <param name="tickType">The tick type used to created the <see cref="SubscriptionDataConfig"/> for the retrieval of history data</param>
/// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
public HistoryRequest(DateTime startTimeUtc,
DateTime endTimeUtc,
Type dataType,
Symbol symbol,
Resolution resolution,
SecurityExchangeHours exchangeHours,
DateTimeZone dataTimeZone,
Resolution?fillForwardResolution,
bool includeExtendedMarketHours,
bool isCustomData,
DataNormalizationMode dataNormalizationMode,
TickType tickType,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
uint contractDepthOffset = 0)
: base(startTimeUtc, endTimeUtc, exchangeHours, tickType)
{
Symbol = symbol;
DataTimeZone = dataTimeZone;
Resolution = resolution;
FillForwardResolution = fillForwardResolution;
IncludeExtendedMarketHours = includeExtendedMarketHours;
DataType = dataType;
IsCustomData = isCustomData;
DataNormalizationMode = dataNormalizationMode;
TickType = tickType;
DataMappingMode = dataMappingMode;
ContractDepthOffset = contractDepthOffset;
}
public void NotEqualsMappingAndOffset(int offsetA, int offsetB, DataMappingMode mappingModeA, DataMappingMode mappingModeB)
{
var configA = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, TimeZones.NewYork,
TimeZones.NewYork, false, false, false, dataMappingMode: mappingModeA, contractDepthOffset: (uint)offsetA);
var configB = new SubscriptionDataConfig(configA, dataMappingMode: mappingModeB, contractDepthOffset: (uint)offsetB);
Assert.AreNotEqual(configA, configB);
Assert.AreNotEqual(configA.GetHashCode(), configB.GetHashCode());
}
/// <summary>
/// Initializes a new instance of the <see cref="UniverseSettings"/> class
/// </summary>
/// <param name="resolution">The resolution</param>
/// <param name="leverage">The leverage to be used</param>
/// <param name="fillForward">True to fill data forward, false otherwise</param>
/// <param name="extendedMarketHours">True to allow extended market hours data, false otherwise</param>
/// <param name="minimumTimeInUniverse">Defines the minimum amount of time a security must remain in the universe before being removed</param>
/// <param name="dataNormalizationMode">Defines how universe data is normalized before being send into the algorithm</param>
/// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest, int contractDepthOffset = 0)
{
Resolution = resolution;
Leverage = leverage;
FillForward = fillForward;
DataMappingMode = dataMappingMode;
ContractDepthOffset = contractDepthOffset;
ExtendedMarketHours = extendedMarketHours;
MinimumTimeInUniverse = minimumTimeInUniverse;
DataNormalizationMode = dataNormalizationMode;
}
/// <summary>
/// Initializes a new instance of the <see cref="UniverseSettings"/> class
/// </summary>
public UniverseSettings(UniverseSettings universeSettings)
{
Resolution = universeSettings.Resolution;
Leverage = universeSettings.Leverage;
FillForward = universeSettings.FillForward;
DataMappingMode = universeSettings.DataMappingMode;
ContractDepthOffset = universeSettings.ContractDepthOffset;
ExtendedMarketHours = universeSettings.ExtendedMarketHours;
MinimumTimeInUniverse = universeSettings.MinimumTimeInUniverse;
DataNormalizationMode = universeSettings.DataNormalizationMode;
SubscriptionDataTypes = universeSettings.SubscriptionDataTypes;
}
/// <summary>
/// Serves as the default hash function.
/// </summary>
/// <returns>
/// A hash code for the current object.
/// </returns>
public override int GetHashCode()
{
unchecked
{
var hashCode = _sid.GetHashCode();
hashCode = (hashCode * 397) ^ Type.GetHashCode();
hashCode = (hashCode * 397) ^ (int)TickType;
hashCode = (hashCode * 397) ^ (int)Resolution;
hashCode = (hashCode * 397) ^ FillDataForward.GetHashCode();
hashCode = (hashCode * 397) ^ ExtendedMarketHours.GetHashCode();
hashCode = (hashCode * 397) ^ IsInternalFeed.GetHashCode();
hashCode = (hashCode * 397) ^ IsCustomData.GetHashCode();
hashCode = (hashCode * 397) ^ DataMappingMode.GetHashCode();
hashCode = (hashCode * 397) ^ DataTimeZone.Id.GetHashCode(); // timezone hash is expensive, use id instead
hashCode = (hashCode * 397) ^ ExchangeTimeZone.Id.GetHashCode(); // timezone hash is expensive, use id instead
hashCode = (hashCode * 397) ^ ContractDepthOffset.GetHashCode();
hashCode = (hashCode * 397) ^ IsFilteredSubscription.GetHashCode();
return(hashCode);
}
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data type to use.
/// If the config already existed will return existing instance instead
/// </summary>
public SubscriptionDataConfig Add(
Type dataType,
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
uint contractDepthOffset = 0
)
{
return(Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))
},
dataNormalizationMode, dataMappingMode, contractDepthOffset)
.First());
}
public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed)
{
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.ES_Future_Chain,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false,
dataMappingMode: dataMappingMode);
var symbolMaps = new List <SubscriptionDataConfig.NewSymbolEventArgs>();
config.NewSymbol += (sender, args) => symbolMaps.Add(args);
var time = new DateTime(2013, 05, 28);
var cache = new SecurityCache();
cache.AddData(new Tick(time, config.Symbol, 20, 10));
var timeProvider = new ManualTimeProvider(time);
var futureTicker1 = "es vhle2yxr5blt";
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new []
{
new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME),
new MapFileRow(new DateTime(2013, 06, 01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth),
new MapFileRow(new DateTime(2013, 06, 15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest),
new MapFileRow(new DateTime(2013, 06, 22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay),
});
IEnumerator <BaseData> enumerator;
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
// get's mapped right away!
Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol);
Assert.AreEqual(1, symbolMaps.Count);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old);
Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New);
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString());
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
if (delayed)
{
// we advance to the mapping date, without any new mapFile!
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
}
else
{
// just advance a day to show nothing happens until mapping time
timeProvider.Advance(TimeSpan.FromDays(1));
}
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var futureTicker2 = "es vk2zrh843z7l";
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat(
new []
{
new MapFileRow(new DateTime(2013, 09, 01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth),
new MapFileRow(new DateTime(2013, 09, 14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest),
new MapFileRow(new DateTime(2013, 09, 21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay),
}));
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
if (delayed)
{
// we got a new mapFile! advance the date and expect mapping to have happened
timeProvider.Advance(TimeSpan.FromDays(1));
}
else
{
// we advance to the mapping date
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
}
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolMaps.Count);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old);
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString());
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New);
Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString());
Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol);
Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol);
Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol);
Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol);
Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
/// <summary>
/// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
/// Can optionally pass in desired subscription data types to use.
/// If the config already existed will return existing instance instead
/// </summary>
public List <SubscriptionDataConfig> Add(
Symbol symbol,
Resolution?resolution = null,
bool fillForward = true,
bool extendedMarketHours = false,
bool isFilteredSubscription = true,
bool isInternalFeed = false,
bool isCustomData = false,
List <Tuple <Type, TickType> > subscriptionDataTypes = null,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
uint contractDepthOffset = 0
)
{
var dataTypes = subscriptionDataTypes ??
LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution ?? Resolution.Minute, symbol.IsCanonical());
if (!dataTypes.Any())
{
throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
}
var resolutionWasProvided = resolution.HasValue;
foreach (var typeTuple in dataTypes)
{
var baseInstance = typeTuple.Item1.GetBaseDataInstance();
baseInstance.Symbol = symbol;
if (!resolutionWasProvided)
{
var defaultResolution = baseInstance.DefaultResolution();
if (resolution.HasValue && resolution != defaultResolution)
{
// we are here because there are multiple 'dataTypes'.
// if we get different default resolutions lets throw, this shouldn't happen
throw new InvalidOperationException(
$"Different data types ({string.Join(",", dataTypes.Select(tuple => tuple.Item1))})" +
$" provided different default resolutions {defaultResolution} and {resolution}, this is an unexpected invalid operation.");
}
resolution = defaultResolution;
}
else
{
// only assert resolution in backtesting, live can use other data source
// for example daily data for options
if (!_liveMode)
{
var supportedResolutions = baseInstance.SupportedResolutions();
if (supportedResolutions.Contains(resolution.Value))
{
continue;
}
throw new ArgumentException($"Sorry {resolution.ToStringInvariant()} is not a supported resolution for {typeTuple.Item1.Name}" +
$" and SecurityType.{symbol.SecurityType.ToStringInvariant()}." +
$" Please change your AddData to use one of the supported resolutions ({string.Join(",", supportedResolutions)}).");
}
}
}
var marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol, dataTypes.Select(tuple => tuple.Item1));
var exchangeHours = marketHoursDbEntry.ExchangeHours;
if (symbol.ID.SecurityType.IsOption() ||
symbol.ID.SecurityType == SecurityType.Index)
{
dataNormalizationMode = DataNormalizationMode.Raw;
}
if (marketHoursDbEntry.DataTimeZone == null)
{
throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
"DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
}
if (exchangeHours.TimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
"ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
}
var result = (from subscriptionDataType in dataTypes
let dataType = subscriptionDataType.Item1
let tickType = subscriptionDataType.Item2
select new SubscriptionDataConfig(
dataType,
symbol,
resolution.Value,
marketHoursDbEntry.DataTimeZone,
exchangeHours.TimeZone,
fillForward,
extendedMarketHours,
// if the subscription data types were not provided and the tick type is OpenInterest we make it internal
subscriptionDataTypes == null && tickType == TickType.OpenInterest || isInternalFeed,
isCustomData,
isFilteredSubscription: isFilteredSubscription,
tickType: tickType,
dataNormalizationMode: dataNormalizationMode,
dataMappingMode: dataMappingMode,
contractDepthOffset: contractDepthOffset)).ToList();
for (int i = 0; i < result.Count; i++)
{
result[i] = SubscriptionManagerGetOrAdd(result[i]);
// track all registered data types
_registeredTypesProvider.RegisterType(result[i].Type);
}
return(result);
}
/// <summary>
/// Constructor for Data Subscriptions
/// </summary>
/// <param name="objectType">Type of the data objects.</param>
/// <param name="symbol">Symbol of the asset we're requesting</param>
/// <param name="resolution">Resolution of the asset we're requesting</param>
/// <param name="dataTimeZone">The time zone the raw data is time stamped in</param>
/// <param name="exchangeTimeZone">Specifies the time zone of the exchange for the security this subscription is for. This
/// is this output time zone, that is, the time zone that will be used on BaseData instances</param>
/// <param name="fillForward">Fill in gaps with historical data</param>
/// <param name="extendedHours">Equities only - send in data from 4am - 8pm</param>
/// <param name="isInternalFeed">Set to true if this subscription is added for the sole purpose of providing currency conversion rates,
/// setting this flag to true will prevent the data from being sent into the algorithm's OnData methods</param>
/// <param name="isCustom">True if this is user supplied custom data, false for normal QC data</param>
/// <param name="tickType">Specifies if trade or quote data is subscribed</param>
/// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
/// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
/// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
public SubscriptionDataConfig(Type objectType,
Symbol symbol,
Resolution resolution,
DateTimeZone dataTimeZone,
DateTimeZone exchangeTimeZone,
bool fillForward,
bool extendedHours,
bool isInternalFeed,
bool isCustom = false,
TickType?tickType = null,
bool isFilteredSubscription = true,
DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
uint contractDepthOffset = 0)
{
if (objectType == null)
{
throw new ArgumentNullException(nameof(objectType));
}
if (symbol == null)
{
throw new ArgumentNullException(nameof(symbol));
}
if (dataTimeZone == null)
{
throw new ArgumentNullException(nameof(dataTimeZone));
}
if (exchangeTimeZone == null)
{
throw new ArgumentNullException(nameof(exchangeTimeZone));
}
Type = objectType;
Resolution = resolution;
_sid = symbol.ID;
Symbol = symbol;
FillDataForward = fillForward;
ExtendedMarketHours = extendedHours;
PriceScaleFactor = 1;
IsInternalFeed = isInternalFeed;
IsCustomData = isCustom;
DataTimeZone = dataTimeZone;
DataMappingMode = dataMappingMode;
ExchangeTimeZone = exchangeTimeZone;
ContractDepthOffset = contractDepthOffset;
IsFilteredSubscription = isFilteredSubscription;
Consolidators = new ConcurrentSet <IDataConsolidator>();
DataNormalizationMode = dataNormalizationMode;
TickType = tickType ?? LeanData.GetCommonTickTypeForCommonDataTypes(objectType, SecurityType);
switch (resolution)
{
case Resolution.Tick:
//Ticks are individual sales and fillforward doesn't apply.
Increment = TimeSpan.FromSeconds(0);
FillDataForward = false;
break;
case Resolution.Second:
Increment = TimeSpan.FromSeconds(1);
break;
case Resolution.Minute:
Increment = TimeSpan.FromMinutes(1);
break;
case Resolution.Hour:
Increment = TimeSpan.FromHours(1);
break;
case Resolution.Daily:
Increment = TimeSpan.FromDays(1);
break;
default:
throw new InvalidEnumArgumentException(Invariant($"Unexpected Resolution: {resolution}"));
}
}