/// <summary>
/// Get the data feed types for a given <see cref="SecurityType"/> <see cref="Resolution"/>
/// </summary>
/// <param name="symbolSecurityType">The <see cref="SecurityType"/> used to determine the types</param>
/// <param name="resolution">The resolution of the data requested</param>
/// <param name="isCanonical">Indicates whether the security is Canonical (future and options)</param>
/// <returns>Types that should be added to the <see cref="SubscriptionDataConfig"/></returns>
public List <Tuple <Type, TickType> > LookupSubscriptionConfigDataTypes(SecurityType symbolSecurityType, Resolution resolution, bool isCanonical)
{
if (isCanonical)
{
return(new List <Tuple <Type, TickType> > {
new Tuple <Type, TickType>(typeof(ZipEntryName), TickType.Quote)
});
}
return(AvailableDataTypes[symbolSecurityType].Select(tickType => new Tuple <Type, TickType>(LeanData.GetDataType(resolution, tickType), tickType)).ToList());
}
/// <summary>
/// Get the last known price using the history provider.
/// Useful for seeding securities with the correct price
/// </summary>
/// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
/// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
public BaseData GetLastKnownPrice(Security security)
{
if (security.Symbol.IsCanonical() || HistoryProvider == null)
{
return(null);
}
var configs = SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol);
var dataTimeZone = MarketHoursDatabase
.GetDataTimeZone(security.Symbol.ID.Market, security.Symbol, security.Symbol.SecurityType);
// For speed and memory usage, use Resolution.Minute as the minimum resolution
var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)configs.GetHighestResolution());
var isExtendedMarketHours = configs.IsExtendedMarketHours();
// request QuoteBar for Options and Futures
var dataType = typeof(BaseData);
if (security.Type == SecurityType.Option || security.Type == SecurityType.Future)
{
dataType = LeanData.GetDataType(resolution, TickType.Quote);
}
// Get the config with the largest resolution
var subscriptionDataConfig = GetMatchingSubscription(security.Symbol, dataType);
TickType tickType;
if (subscriptionDataConfig == null)
{
dataType = typeof(TradeBar);
tickType = LeanData.GetCommonTickTypeForCommonDataTypes(dataType, security.Type);
}
else
{
// if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar
if (subscriptionDataConfig.Resolution == Resolution.Tick)
{
dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType);
subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution);
}
dataType = subscriptionDataConfig.Type;
tickType = subscriptionDataConfig.TickType;
}
Func <int, BaseData> getLastKnownPriceForPeriods = backwardsPeriods =>
{
var startTimeUtc = _historyRequestFactory
.GetStartTimeAlgoTz(security.Symbol, backwardsPeriods, resolution, security.Exchange.Hours)
.ConvertToUtc(_localTimeKeeper.TimeZone);
var request = new HistoryRequest(
startTimeUtc,
UtcTime,
dataType,
security.Symbol,
resolution,
security.Exchange.Hours,
dataTimeZone,
resolution,
isExtendedMarketHours,
configs.IsCustomData(),
configs.DataNormalizationMode(),
tickType
);
BaseData result = null;
History(new List <HistoryRequest> {
request
})
.PushThrough(bar =>
{
if (!bar.IsFillForward)
{
result = bar;
}
});
return(result);
};
var lastKnownPrice = getLastKnownPriceForPeriods(1);
if (lastKnownPrice != null)
{
return(lastKnownPrice);
}
// If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
// We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
var periods =
resolution == Resolution.Daily ? 3 :
resolution == Resolution.Hour ? 24 : 1440;
return(getLastKnownPriceForPeriods(periods));
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary <Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24) },
{ Resolution.Hour, TimeSpan.FromHours(1) },
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >()
{
{ TickType.Quote, () => new QuoteBarConsolidator(timeSpan) },
{ TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) },
{ TickType.Trade, () => new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary <string, IDataConsolidator>();
var configs = new Dictionary <string, SubscriptionDataConfig>();
var outputFiles = new Dictionary <string, StringBuilder>();
var futures = new Dictionary <string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var data = leanDataReader.Parse().ToList();
var consolidator = consolidators[future.Value];
foreach (var bar in data)
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
{
consolidator.Scan(date);
}
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}
/// <summary>
/// Downloads historical data from the brokerage and saves it in LEAN format.
/// </summary>
/// <param name="brokerage">The brokerage from where to fetch the data</param>
/// <param name="symbols">The list of symbols</param>
/// <param name="startTimeUtc">The starting date/time (UTC)</param>
/// <param name="endTimeUtc">The ending date/time (UTC)</param>
public void DownloadAndSave(IBrokerage brokerage, List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc)
{
if (symbols.Count == 0)
{
throw new ArgumentException("DownloadAndSave(): The symbol list cannot be empty.");
}
if (_tickType != TickType.Trade && _tickType != TickType.Quote)
{
throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote.");
}
if (_securityType != SecurityType.Future && _securityType != SecurityType.Option)
{
throw new ArgumentException($"DownloadAndSave(): The security type must be {SecurityType.Future} or {SecurityType.Option}.");
}
if (symbols.Any(x => x.SecurityType != _securityType))
{
throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type.");
}
if (symbols.DistinctBy(x => x.ID.Symbol).Count() > 1)
{
throw new ArgumentException("DownloadAndSave(): All symbols must have the same root ticker.");
}
var dataType = LeanData.GetDataType(_resolution, _tickType);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var ticker = symbols.First().ID.Symbol;
var market = symbols.First().ID.Market;
var canonicalSymbol = Symbol.Create(ticker, _securityType, market);
var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
var historyBySymbol = new Dictionary <Symbol, List <IGrouping <DateTime, BaseData> > >();
var historyBySymbolDailyOrHour = new Dictionary <Symbol, List <BaseData> >();
foreach (var symbol in symbols)
{
var historyRequest = new HistoryRequest(
startTimeUtc,
endTimeUtc,
dataType,
symbol,
_resolution,
exchangeHours,
dataTimeZone,
_resolution,
true,
false,
DataNormalizationMode.Raw,
_tickType
);
var history = brokerage.GetHistory(historyRequest)
.Select(
x =>
{
x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
return(x);
})
.ToList();
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
historyBySymbolDailyOrHour.Add(symbol, history);
}
else
{
// group by date in DataTimeZone
var historyByDate = history.GroupBy(x => x.Time.Date).ToList();
historyBySymbol.Add(symbol, historyByDate);
}
}
if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
{
SaveDailyOrHour(symbols, canonicalSymbol, historyBySymbolDailyOrHour);
}
else
{
SaveMinuteOrSecondOrTick(symbols, startTimeUtc, endTimeUtc, canonicalSymbol, historyBySymbol);
}
}
public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool shouldBeEmpty)
{
var now = new DateTime(2020, 5, 20, 15, 0, 0).RoundDown(resolution.ToTimeSpan());
var dataType = LeanData.GetDataType(resolution, tickType);
var requests = new[]
{
new HistoryRequest(now.Add(-period),
now,
dataType,
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
true,
false,
DataNormalizationMode.Adjusted,
tickType)
};
var history = _historyProvider.GetHistory(requests, TimeZones.NewYork).ToList();
if (dataType == typeof(TradeBar))
{
foreach (var slice in history)
{
var bar = slice.Bars[symbol];
Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}");
}
}
else if (dataType == typeof(QuoteBar))
{
foreach (var slice in history)
{
var bar = slice.QuoteBars[symbol];
Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}");
}
}
else if (dataType == typeof(Tick))
{
foreach (var slice in history)
{
var ticks = slice.Ticks[symbol];
foreach (var tick in ticks)
{
Log.Trace($"{tick.Time}: {tick.Symbol} - B={tick.BidPrice}, A={tick.AskPrice}, P={tick.LastPrice}, Q={tick.Quantity}");
}
}
}
Log.Trace("Data points retrieved: " + _historyProvider.DataPointCount);
if (shouldBeEmpty)
{
Assert.IsTrue(history.Count == 0);
}
else
{
Assert.IsTrue(history.Count > 0);
}
}
/// <summary>
/// Get the last known price using the history provider.
/// Useful for seeding securities with the correct price
/// </summary>
/// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
/// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
public BaseData GetLastKnownPrice(Security security)
{
if (security.Symbol.IsCanonical() || HistoryProvider == null)
{
return(null);
}
var configs = SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol);
// For speed and memory usage, use Resolution.Minute as the minimum resolution
var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)configs.GetHighestResolution());
var isExtendedMarketHours = configs.IsExtendedMarketHours();
var startTime = GetStartTimeAlgoTzForSecurity(security.Exchange, 1, resolution, isExtendedMarketHours);
var endTime = Time.RoundDown(resolution.ToTimeSpan());
// request QuoteBar for Options and Futures
var dataType = typeof(BaseData);
if (security.Type == SecurityType.Option || security.Type == SecurityType.Future)
{
dataType = LeanData.GetDataType(resolution, TickType.Quote);
}
// Get the config with the largest resolution
var subscriptionDataConfig = GetMatchingSubscription(security.Symbol, dataType);
// if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar
if (subscriptionDataConfig != null && subscriptionDataConfig.Resolution == Resolution.Tick)
{
dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType);
subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution);
}
var request = new HistoryRequest(
startTime.ConvertToUtc(_localTimeKeeper.TimeZone),
endTime.ConvertToUtc(_localTimeKeeper.TimeZone),
subscriptionDataConfig == null ? typeof(TradeBar) : subscriptionDataConfig.Type,
security.Symbol,
resolution,
security.Exchange.Hours,
MarketHoursDatabase.FromDataFolder().GetDataTimeZone(security.Symbol.ID.Market, security.Symbol, security.Symbol.SecurityType),
resolution,
isExtendedMarketHours,
configs.IsCustomData(),
configs.DataNormalizationMode(),
subscriptionDataConfig == null ? LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), security.Type) : subscriptionDataConfig.TickType
);
var history = History(new List <HistoryRequest> {
request
}).ToList();
if (history.Any() && history.First().Values.Any())
{
return(history.First().Values.First());
}
return(null);
}
