public virtual DateTime NextTickTime(Symbol symbol, DateTime previous, Resolution resolution, DataDensity density)
{
var increment = resolution.ToTimeSpan();
if (increment == TimeSpan.Zero)
{
increment = TimeSpan.FromMilliseconds(500);
}
double steps;
switch (density)
{
case DataDensity.Dense:
steps = 0.5 * _random.NextDouble();
break;
case DataDensity.Sparse:
steps = 5 * _random.NextDouble();
break;
case DataDensity.VerySparse:
steps = 50 * _random.NextDouble();
break;
default:
throw new ArgumentOutOfRangeException(nameof(density), density, null);
}
var delta = TimeSpan.FromTicks((long)(steps * increment.Ticks));
var tickTime = previous.Add(delta);
if (tickTime == previous)
{
tickTime = tickTime.Add(increment);
}
var barStart = tickTime.Subtract(increment);
var marketHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
if (!marketHours.IsDateOpen(tickTime) || !marketHours.IsOpen(barStart, tickTime, false))
{
// we ended up outside of market hours, emit a new tick at market open
var nextMarketOpen = marketHours.GetNextMarketOpen(tickTime, false);
if (resolution == Resolution.Tick)
{
resolution = Resolution.Second;
}
// emit a new tick somewhere in the next trading day at a step higher resolution to guarantee a hit
return(NextTickTime(symbol, nextMarketOpen, resolution - 1, density));
}
return(tickTime);
}
private Insight SetGeneratedAndClosedTimes(Insight insight)
{
insight.GeneratedTimeUtc = UtcTime;
insight.ReferenceValue = _securityValuesProvider.GetValues(insight.Symbol).Get(insight.Type);
if (string.IsNullOrEmpty(insight.SourceModel))
{
// set the source model name if not already set
insight.SourceModel = Alpha.GetModelName();
}
TimeSpan barSize;
Security security;
SecurityExchangeHours exchangeHours;
if (Securities.TryGetValue(insight.Symbol, out security))
{
exchangeHours = security.Exchange.Hours;
barSize = security.Resolution.ToTimeSpan();
}
else
{
barSize = insight.Period.ToHigherResolutionEquivalent(false).ToTimeSpan();
exchangeHours = MarketHoursDatabase.GetExchangeHours(insight.Symbol.ID.Market, insight.Symbol, insight.Symbol.SecurityType);
}
var localStart = UtcTime.ConvertFromUtc(exchangeHours.TimeZone);
barSize = QuantConnect.Time.Max(barSize, QuantConnect.Time.OneMinute);
var barCount = (int)(insight.Period.Ticks / barSize.Ticks);
insight.CloseTimeUtc = QuantConnect.Time.GetEndTimeForTradeBars(exchangeHours, localStart, barSize, barCount, false).ConvertToUtc(exchangeHours.TimeZone);
return(insight);
}
private Alpha SetGeneratedAndClosedTimes(Alpha alpha)
{
alpha.GeneratedTimeUtc = UtcTime;
TimeSpan barSize;
Security security;
SecurityExchangeHours exchangeHours;
if (Securities.TryGetValue(alpha.Symbol, out security))
{
exchangeHours = security.Exchange.Hours;
barSize = security.Resolution.ToTimeSpan();
}
else
{
barSize = alpha.Period.ToHigherResolutionEquivalent(false).ToTimeSpan();
exchangeHours = MarketHoursDatabase.GetExchangeHours(alpha.Symbol.ID.Market, alpha.Symbol, alpha.Symbol.SecurityType);
}
var localStart = UtcTime.ConvertFromUtc(exchangeHours.TimeZone);
barSize = QuantConnect.Time.Max(barSize, QuantConnect.Time.OneMinute);
var barCount = (int)(alpha.Period.Ticks / barSize.Ticks);
alpha.CloseTimeUtc = QuantConnect.Time.GetEndTimeForTradeBars(exchangeHours, localStart, barSize, barCount, false).ConvertToUtc(exchangeHours.TimeZone);
return(alpha);
}
/// <summary>
/// Returns an enumerator that defines when this user defined universe will be invoked
/// </summary>
/// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns>
public virtual IEnumerable <DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase)
{
var exchangeHours = marketHoursDatabase.GetExchangeHours(Configuration);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var first = true;
foreach (var dateTime in LinqExtensions.Range(localStartTime, localEndTime, dt => dt + Interval))
{
//Console.WriteLine("------------DATE TIME-------------------------------");
//Console.WriteLine(localStartTime + " " + localEndTime + " " + Interval);
//Console.WriteLine(dateTime);
if (first)
{
yield return(dateTime);
first = false;
}
else if (exchangeHours.IsOpen(dateTime, dateTime + Interval, Configuration.ExtendedMarketHours))
{
yield return(dateTime);
}
}
}
/// <summary>
/// Gets the history for the requested security
/// </summary>
/// <param name="request">The historical data request</param>
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable <BaseData> GetHistory(HistoryRequest request)
{
var exchangeTimeZone = _marketHours.GetExchangeHours(request.Symbol.ID.Market, request.Symbol, request.Symbol.SecurityType).TimeZone;
IEnumerable <BaseData> items;
switch (request.Resolution)
{
case Resolution.Tick:
items = DownloadTradeTicks(request.Symbol, request.StartTimeUtc, request.EndTimeUtc, exchangeTimeZone);
break;
case Resolution.Second:
var ticks = DownloadTradeTicks(request.Symbol, request.StartTimeUtc, request.EndTimeUtc, exchangeTimeZone);
items = AggregateTicks(request.Symbol, ticks, request.Resolution.ToTimeSpan());
break;
default:
items = DownloadTradeBars(request.Symbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, exchangeTimeZone);
break;
}
foreach (var item in items)
{
yield return(item);
}
}
/// <summary>
/// Helper class used to set values not required to be set by alpha models
/// </summary>
/// <param name="insight">The <see cref="Insight"/> to set the values for</param>
/// <returns>The same <see cref="Insight"/> instance with the values set</returns>
private Insight InitializeInsightFields(Insight insight)
{
insight.GeneratedTimeUtc = UtcTime;
insight.ReferenceValue = _securityValuesProvider.GetValues(insight.Symbol).Get(insight.Type);
insight.SourceModel = string.IsNullOrEmpty(insight.SourceModel) ? Alpha.GetModelName() : insight.SourceModel;
var exchangeHours = MarketHoursDatabase.GetExchangeHours(insight.Symbol.ID.Market, insight.Symbol, insight.Symbol.SecurityType);
insight.SetPeriodAndCloseTime(exchangeHours);
return(insight);
}
private TimeZoneOffsetProvider GetTimeZoneOffsetProvider(Symbol symbol)
{
TimeZoneOffsetProvider offsetProvider;
if (!_symbolExchangeTimeZones.TryGetValue(symbol, out offsetProvider))
{
// read the exchange time zone from market-hours-database
var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
_symbolExchangeTimeZones[symbol] = offsetProvider = new TimeZoneOffsetProvider(exchangeTimeZone, TimeProvider.GetUtcNow(), Time.EndOfTime);
}
return(offsetProvider);
}
private bool TryGetWarmupHistoryStartTime(out DateTime result)
{
result = Time;
if (_warmupBarCount.HasValue)
{
var symbols = Securities.Keys;
if (symbols.Count != 0)
{
var startTimeUtc = CreateBarCountHistoryRequests(symbols, _warmupBarCount.Value, _warmupResolution)
.DefaultIfEmpty()
.Min(request => request == null ? default : request.StartTimeUtc);
if (startTimeUtc != default)
{
result = startTimeUtc.ConvertFromUtc(TimeZone);
return(true);
}
}
var defaultResolutionToUse = UniverseSettings.Resolution;
if (_warmupResolution.HasValue)
{
defaultResolutionToUse = _warmupResolution.Value;
}
// if the algorithm has no added security, let's take a look at the universes to determine
// what the start date should be used. Defaulting to always open
result = Time - _warmupBarCount.Value * defaultResolutionToUse.ToTimeSpan();
foreach (var universe in _pendingUniverseAdditions.Concat(UniverseManager.Values))
{
var config = universe.Configuration;
var resolution = universe.Configuration.Resolution;
if (_warmupResolution.HasValue)
{
resolution = _warmupResolution.Value;
}
var exchange = MarketHoursDatabase.GetExchangeHours(config);
var start = _historyRequestFactory.GetStartTimeAlgoTz(config.Symbol, _warmupBarCount.Value, resolution, exchange, config.DataTimeZone);
// we choose the min start
result = result < start ? result : start;
}
return(true);
}
if (_warmupTimeSpan.HasValue)
{
result = Time - _warmupTimeSpan.Value;
return(true);
}
return(false);
}
private DateTime GetTickTime(Symbol symbol, DateTime utcTime)
{
DateTimeZone exchangeTimeZone;
if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone))
{
// read the exchange time zone from market-hours-database
exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
_symbolExchangeTimeZones.Add(symbol, exchangeTimeZone);
}
return(utcTime.ConvertFromUtc(exchangeTimeZone));
}
/// <summary>
/// Generates a new random future <see cref="Symbol"/>. The generates future contract Symbol will have an
/// expiry between the specified minExpiry and maxExpiry.
/// </summary>
/// <param name="ticker">Optionally can provide a ticker that should be used</param>
/// <returns>A new future contract Symbol with the specified expiration parameters</returns>
protected override IEnumerable <Symbol> GenerateAsset(string ticker = null)
{
if (ticker == null)
{
// get a valid ticker from the Symbol properties database
ticker = NextTickerFromSymbolPropertiesDatabase(SecurityType.Future, _market);
}
var marketHours = MarketHoursDatabase.GetExchangeHours(_market, ticker, SecurityType.Future);
var expiry = GetRandomExpiration(marketHours, _minExpiry, _maxExpiry);
yield return(Symbol.CreateFuture(ticker, _market, expiry));
}
/// <summary>
/// Event invocator for the <see cref="QCAlgorithm.InsightsGenerated"/> event
/// </summary>
/// <remarks>
/// This method is sealed because the framework must be able to force setting of the
/// generated and close times before any event handlers are run. Bind directly to the
/// <see cref="QCAlgorithm.InsightsGenerated"/> event insead of overriding.
/// </remarks>
/// <param name="insights">The collection of insights generaed at the current time step</param>
protected sealed override void OnInsightsGenerated(IEnumerable <Insight> insights)
{
// set values not required to be set by alpha models
base.OnInsightsGenerated(insights.Select(insight =>
{
insight.GeneratedTimeUtc = UtcTime;
insight.ReferenceValue = _securityValuesProvider.GetValues(insight.Symbol).Get(insight.Type);
insight.SourceModel = string.IsNullOrEmpty(insight.SourceModel) ? Alpha.GetModelName() : insight.SourceModel;
var exchangeHours = MarketHoursDatabase.GetExchangeHours(insight.Symbol.ID.Market, insight.Symbol, insight.Symbol.SecurityType);
insight.SetPeriodAndCloseTime(exchangeHours);
return(insight);
}));
}
/// <summary>
/// Get the calendar open hours for the date.
/// </summary>
private IEnumerable <MarketHoursSegment> MarketToday(DateTime time, Symbol symbol)
{
if (Config.GetBool("force-exchange-always-open"))
{
yield return(MarketHoursSegment.OpenAllDay());
yield break;
}
var hours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.ID.SecurityType);
foreach (var segment in hours.MarketHours[time.DayOfWeek].Segments)
{
yield return(segment);
}
}
/// <summary>
/// Run the reconciliation for a contract
/// </summary>
/// <param name="contractDetails"></param>
/// <param name="symbol"></param>
public void CheckOpenings(ContractDetails contractDetails, Symbol symbol)
{
if (contractDetails == null || symbol == null)
{
return;
}
var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var sourceTz = DateTimeZoneProviders.Tzdb[contractDetails.TimeZoneId];
var ibTradingTimes =
ConvertTimeZone(
PatchTradingTimes(
ParseTradingTimes(contractDetails.TradingHours),
symbol.ID.Symbol),
sourceTz, exchangeHours.TimeZone)
.ToArray();
var start = ibTradingTimes[0].Open.AddSeconds(-10);
var warnings = new List <string>();
foreach (var(Open, Close) in ibTradingTimes)
{
var expectedOpen = exchangeHours.GetNextMarketOpen(start, _extendedHours);
if (expectedOpen != Open)
{
warnings.Add($"Market hours mismatch for {symbol.ID.Symbol} expected open: {expectedOpen}[{exchangeHours.TimeZone.Id}], actual open: {Open}[{contractDetails.TimeZoneId}]");
}
var expectedClose = exchangeHours.GetNextMarketClose(expectedOpen, _extendedHours);
while (expectedClose.Hour == 0 &&
expectedClose.Minute == 0 &&
exchangeHours.GetNextMarketOpen(expectedClose.AddSeconds(-10), _extendedHours) == expectedClose)
{
// potential overnight
expectedClose = exchangeHours.GetNextMarketClose(expectedClose, _extendedHours);
}
if (expectedClose != Close)
{
warnings.Add($"Market hours mismatch for {symbol.ID.Symbol} expected close: {expectedClose}[{exchangeHours.TimeZone.Id}], actual close: {Close}[{contractDetails.TimeZoneId}]");
}
start = Close.AddSeconds(10);
}
if (warnings.Count > 0)
{
PublishWarning($"{symbol.ID.Symbol}: {string.Join(Environment.NewLine, warnings)}");
}
}
/// <summary>
/// From https://stackoverflow.com/questions/1617049/calculate-the-number-of-business-days-between-two-dates
/// </summary>
private int GetBusinessDays(DateTime start, DateTime end, Symbol symbol)
{
var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var current = start.Date;
var days = 0;
while (current < end)
{
if (exchangeHours.IsDateOpen(current))
{
days++;
}
current = current.AddDays(1);
}
return(days);
}
/// <summary>
/// Generates a new random option <see cref="Symbol"/>. The generated option contract Symbol will have an
/// expiry between the specified min and max expiration. The strike
/// price will be within the specified maximum strike price deviation of the underlying symbol price
/// and should be rounded to reasonable value for the given price. For example, a price of 100 dollars would round
/// to 5 dollar increments and a price of 5 dollars would round to 50 cent increments
/// </summary>
/// <param name="ticker">Optionally can provide a ticker that should be used</param>
/// <remarks>
/// Standard contracts expiry on the third Friday.
/// Weekly contracts expiry every week on Friday
/// </remarks>
/// <returns>A new option contract Symbol within the specified expiration and strike price parameters along with its underlying symbol</returns>
protected override IEnumerable <Symbol> GenerateAsset(string ticker = null)
{
// first generate the underlying
var underlying = NextSymbol(_underlyingSecurityType, _market, ticker);
yield return(underlying);
var marketHours = MarketHoursDatabase.GetExchangeHours(_market, underlying, _underlyingSecurityType);
var expiry = GetRandomExpiration(marketHours, _minExpiry, _maxExpiry);
var strikes = new HashSet <decimal>();
for (var i = 0; i < _symbolChainSize; i++)
{
decimal strike;
do
{
// generate a random strike while respecting the maximum deviation from the underlying's price
// since these are underlying prices, use Equity as the security type
strike = Random.NextPrice(_underlyingSecurityType, _market, _underlyingPrice,
_maximumStrikePriceDeviation);
// round the strike price to something reasonable
var order = 1 + Math.Log10((double)strike);
strike = strike.RoundToSignificantDigits((int)order);
}
// don't allow duplicate strikes
while (!strikes.Add(strike));
foreach (var optionRight in new [] { OptionRight.Put, OptionRight.Call })
{
// when providing a null option w/ an expiry, it will automatically create the OSI ticker string for the Value
yield return(Symbol.CreateOption(underlying, _market, underlying.SecurityType.DefaultOptionStyle(), optionRight, strike, expiry));
}
}
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
var liveStart = DateTime.UtcNow;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.LiveMode ? liveStart : algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[CashBook.AccountCurrency],
SymbolProperties.GetDefault(CashBook.AccountCurrency),
algorithm.Portfolio.CashBook
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase,
bool liveMode,
IRegisteredSecurityDataTypesProvider registeredTypesProvider,
IDataPermissionManager dataPermissionManager)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
_liveMode = liveMode;
_registeredTypesProvider = registeredTypesProvider;
_dataPermissionManager = dataPermissionManager;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
universe.Dispose();
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}