/// <summary>
/// Initializes a new instance of the <see cref="UserDefinedUniverse"/> class
/// </summary>
/// <param name="configuration">The configuration used to resolve the data for universe selection</param>
/// <param name="subscriptionSettings">The settings used for new subscriptions generated by this universe</param>
/// <param name="interval">The interval at which selection should be performed</param>
/// <param name="symbols">The initial set of symbols in this universe</param>
public UserDefinedUniverse(SubscriptionDataConfig configuration, SubscriptionSettings subscriptionSettings, TimeSpan interval, IEnumerable<Symbol> symbols)
: base(configuration)
{
_interval = interval;
_symbols = symbols.ToHashSet();
_subscriptionSettings = subscriptionSettings;
}
/// <summary>
/// Generic Reader Implementation for Quandl Data.
/// </summary>
/// <param name="config">Subscription configuration</param>
/// <param name="line">CSV line of data from the souce</param>
/// <param name="date">Date of the requested line</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns></returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var data = new Quandl();
data.Symbol = config.Symbol;
var csv = line.Split(',');
if (!_isInitialized)
{
_isInitialized = true;
foreach (var propertyName in csv)
{
var property = propertyName.TrimStart().TrimEnd();
// should we remove property names like Time?
// do we need to alias the Time??
data.SetProperty(property, 0m);
_propertyNames.Add(property);
}
return data;
}
data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
for (var i = 1; i < csv.Length; i++)
{
var value = csv[i].ToDecimal();
data.SetProperty(_propertyNames[i], value);
}
// we know that there is a close property, we want to set that to 'Value'
data.Value = (decimal)data.GetProperty(_valueColumn);
return data;
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called.
/// </summary>
/// <param name="config">Subscription data config setup object</param>
/// <param name="line">Line of the source document</param>
/// <param name="date">Date of the requested data</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>Instance of the T:BaseData object generated by this line of the CSV</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
try
{
var csv = line.Split(',');
var coarse = new CoarseFundamental
{
Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]),
Time = date,
Market = config.Market,
Value = csv[2].ToDecimal(),
Volume = csv[3].ToInt64(),
DollarVolume = csv[4].ToDecimal()
};
if (csv.Length > 5)
{
coarse.HasFundamentalData = Convert.ToBoolean(csv[5]);
}
return coarse;
}
catch (Exception)
{
return null;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="UserDefinedUniverse"/> class
/// </summary>
/// <param name="configuration">The configuration used to resolve the data for universe selection</param>
/// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
/// <param name="securityInitializer">Initializes securities when they're added to the universe</param>
/// <param name="interval">The interval at which selection should be performed</param>
/// <param name="symbols">The initial set of symbols in this universe</param>
public UserDefinedUniverse(SubscriptionDataConfig configuration, UniverseSettings universeSettings, ISecurityInitializer securityInitializer, TimeSpan interval, IEnumerable<Symbol> symbols)
: base(configuration, securityInitializer)
{
_interval = interval;
_symbols = symbols.ToHashSet();
_universeSettings = universeSettings;
_selector = time => _symbols;
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var source =
Path.Combine(Globals.DataFolder, "equity", config.Market, "fundamental", "fine",
config.Symbol.Value.ToLower(), date.ToString("yyyyMMdd") + ".zip");
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
}
/// <summary>
/// Initializes a new instance of the <see cref="Universe"/> class
/// </summary>
/// <param name="config">The configuration used to source data for this universe</param>
protected Universe(SubscriptionDataConfig config)
{
_securities = new ConcurrentDictionary<Symbol, Security>();
if (config.FillDataForward) throw new ArgumentException("Universe data can not be fill forward.");
if (!config.IsInternalFeed) throw new ArgumentException("Universe data must be marked as internal feed.");
Configuration = config;
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
return new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile);
}
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName);
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called. The returned object is assumed to be time stamped in the config.ExchangeTimeZone.
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var data = JsonConvert.DeserializeObject<FineFundamental>(line);
data.DataType = MarketDataType.Auxiliary;
data.Symbol = config.Symbol;
data.Time = date;
return data;
}
/// <summary>
/// Initializes a new instance of the <see cref="UserDefinedUniverse"/> class
/// </summary>
/// <param name="configuration">The configuration used to resolve the data for universe selection</param>
/// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
/// <param name="securityInitializer">Initializes securities when they're added to the universe</param>
/// <param name="interval">The interval at which selection should be performed</param>
/// <param name="selector">Universe selection function invoked for each time returned via GetTriggerTimes.
/// The function parameter is a DateTime in the time zone of configuration.ExchangeTimeZone</param>
public UserDefinedUniverse(SubscriptionDataConfig configuration, UniverseSettings universeSettings, ISecurityInitializer securityInitializer, TimeSpan interval, Func<DateTime,IEnumerable<string>> selector)
: base(configuration, securityInitializer)
{
_interval = interval;
_universeSettings = universeSettings;
_selector = time =>
{
var selectSymbolsResult = selector(time.ConvertFromUtc(Configuration.ExchangeTimeZone));
// if we received an unchaged then short circuit the symbol creation and return it directly
if (ReferenceEquals(selectSymbolsResult, Unchanged)) return Unchanged;
return selectSymbolsResult.Select(sym => Symbol.Create(sym, Configuration.SecurityType, Configuration.Market));
};
}
/// <summary>
/// Parse a line from CSV data sources into our trade bars.
/// </summary>
/// <param name="config">Configuration class object for this data subscription</param>
/// <param name="baseDate">Base date of this tradebar line</param>
/// <param name="line">CSV line from source data file</param>
/// <param name="datafeed">Datafeed this csv line is sourced from (backtesting or live)</param>
public TradeBar(SubscriptionDataConfig config, string line, DateTime baseDate, DataFeedEndpoint datafeed = DataFeedEndpoint.Backtesting)
{
try
{
Period = config.Resolution.ToTimeSpan();
//Parse the data into a trade bar:
var csv = line.Split(',');
const decimal scaleFactor = 10000m;
Symbol = config.Symbol;
switch (config.SecurityType)
{
//Equity File Data Format:
case SecurityType.Equity:
Time = baseDate.Date.AddMilliseconds(Convert.ToInt32(csv[0]));
Open = config.GetNormalizedPrice(csv[1].ToDecimal() / scaleFactor); // Convert.ToDecimal(csv[1]) / scaleFactor;
High = config.GetNormalizedPrice(csv[2].ToDecimal() / scaleFactor); // Using custom "ToDecimal" conversion for speed.
Low = config.GetNormalizedPrice(csv[3].ToDecimal() / scaleFactor);
Close = config.GetNormalizedPrice(csv[4].ToDecimal() / scaleFactor);
Volume = Convert.ToInt64(csv[5]);
break;
//FOREX has a different data file format:
case SecurityType.Forex:
Time = DateTime.ParseExact(csv[0], "yyyyMMdd HH:mm:ss.ffff", CultureInfo.InvariantCulture);
Open = csv[1].ToDecimal();
High = csv[2].ToDecimal();
Low = csv[3].ToDecimal();
Close = csv[4].ToDecimal();
break;
}
//base.Value = Close;
}
catch (Exception err)
{
Log.Error("DataModels: TradeBar(): Error Initializing - " + config.SecurityType + " - " + err.Message + " - " + line);
}
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.SecurityType,
request.Symbol,
request.Resolution,
request.Market,
request.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, 1.0m);
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, start, end);
return(new Subscription(null, security, reader, timeZoneOffsetProvider, start, end, false));
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataCacheProvider">Used to cache data</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <param name="factory">The base data instance factory</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode, BaseData factory)
{
ISubscriptionDataSourceReader reader;
TextSubscriptionDataSourceReader textReader = null;
switch (source.Format)
{
case FileFormat.Csv:
reader = textReader = new TextSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
break;
case FileFormat.Collection:
reader = new CollectionSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
break;
case FileFormat.ZipEntryName:
reader = new ZipEntryNameSubscriptionDataSourceReader(config, date, isLiveMode);
break;
case FileFormat.Index:
return(new IndexSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode));
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
// wire up event handlers for logging missing files
if (source.TransportMedium == SubscriptionTransportMedium.LocalFile)
{
if (!factory.IsSparseData())
{
reader.InvalidSource += (sender, args) => Log.Error($"SubscriptionDataSourceReader.InvalidSource(): File not found: {args.Source.Source}");
if (textReader != null)
{
textReader.CreateStreamReaderError += (sender, args) => Log.Error($"SubscriptionDataSourceReader.CreateStreamReaderError(): File not found: {args.Source.Source}");
}
}
}
return(reader);
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String source location of the file</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
// we have a design in github for options structure: https://github.com/QuantConnect/Lean/issues/166
throw new NotImplementedException("QuoteBar folder structure has not been implemented yet.");
}
/// <summary>
/// Tick implementation of reader method: read a line of data from the source and convert it to a tick object.
/// </summary>
/// <param name="config">Subscription configuration object for algorithm</param>
/// <param name="line">Line from the datafeed source</param>
/// <param name="date">Date of this reader request</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>New Initialized tick</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
// currently ticks don't come through the reader function
return new Tick();
}
return new Tick(config, line, date);
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
// this is really a base security type, but we're pulling data from the equities/forex
config.Security = _type;
var file = base.GetSource(config, date, datafeed);
return file;
}
private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false,
SecurityType type = SecurityType.Equity, TickType tickType = TickType.Trade)
{
var start = DateTime.UtcNow;
var end = start.AddSeconds(10);
Security security;
Symbol _symbol;
if (type == SecurityType.Equity)
{
_symbol = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol);
security = new Equity(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
else if (type == SecurityType.Option)
{
_symbol = new Symbol(SecurityIdentifier.GenerateOption(DateTime.Now,
SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA),
Market.USA, 0.0m, OptionRight.Call, OptionStyle.American), symbol);
security = new Option(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
else if (type == SecurityType.Future)
{
_symbol = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.USA), symbol);
security = new Future(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
else
{
throw new Exception("SecurityType not implemented");
}
var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
var enumerator = new EnqueueableEnumerator <BaseData>();
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called.
/// </summary>
/// <param name="config">Subscription data config setup object</param>
/// <param name="line">Line of the source document</param>
/// <param name="date">Date of the requested data</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>Instance of the T:BaseData object generated by this line of the CSV</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
try
{
var csv = line.Split(',');
return new CoarseFundamental
{
Symbol = new Symbol(csv[0], csv[1]),
Time = date,
Market = config.Market,
Value = csv[2].ToDecimal(),
Volume = csv[3].ToInt64(),
DollarVolume = csv[4].ToDecimal()
};
}
catch (Exception)
{
return null;
}
}
/// <summary>
/// Performs a lookup using the specified information and returns the exchange hours if found,
/// if exchange hours are not found, an exception is thrown
/// </summary>
/// <param name="configuration">The subscription data config to get exchange hours for</param>
/// <param name="overrideTimeZone">Specify this time zone to override the resolved time zone from the market hours database.
/// This value will also be used as the time zone for SecurityType.Base with no market hours database entry.
/// If null is specified, no override will be performed. If null is specified, and it's SecurityType.Base, then Utc will be used.</param>
public SecurityExchangeHours GetExchangeHours(SubscriptionDataConfig configuration, DateTimeZone overrideTimeZone = null)
{
return(GetExchangeHours(configuration.Market, configuration.Symbol, configuration.SecurityType, overrideTimeZone));
}
public void RefreshesOptionChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 19, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
TradeBarBuilderEnumerator underlyingEnumerator = null;
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
underlyingEnumerator = (TradeBarBuilderEnumerator)input;
return(new LiveFillForwardEnumerator(
timeProvider,
input,
option.Exchange,
fillForwardResolution,
false,
Time.EndOfTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
Time.BeginningOfTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new OptionChainUniverse(option, universeSettings, true);
var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, new DefaultDataProvider());
// 2018-10-19 10:00 AM UTC
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
// no underlying data available yet
Assert.IsNull(enumerator.Current);
Assert.AreEqual(0, symbolUniverse.TotalLookupCalls);
// 2018-10-19 10:01 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:02 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:03 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:03 AM UTC
timeProvider.Advance(Time.OneDay);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:04 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
enumerator.Dispose();
}
/// <summary>
/// Get the last known price using the history provider.
/// Useful for seeding securities with the correct price
/// </summary>
/// <param name="security"><see cref="Security"/> object for which to retrieve historical data</param>
/// <returns>A single <see cref="BaseData"/> object with the last known price</returns>
public BaseData GetLastKnownPrice(Security security)
{
if (security.Symbol.IsCanonical() || HistoryProvider == null)
{
return(null);
}
var configs = SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol);
var dataTimeZone = MarketHoursDatabase
.GetDataTimeZone(security.Symbol.ID.Market, security.Symbol, security.Symbol.SecurityType);
// For speed and memory usage, use Resolution.Minute as the minimum resolution
var resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)configs.GetHighestResolution());
var isExtendedMarketHours = configs.IsExtendedMarketHours();
// request QuoteBar for Options and Futures
var dataType = typeof(BaseData);
if (security.Type == SecurityType.Option || security.Type == SecurityType.Future)
{
dataType = LeanData.GetDataType(resolution, TickType.Quote);
}
// Get the config with the largest resolution
var subscriptionDataConfig = GetMatchingSubscription(security.Symbol, dataType);
TickType tickType;
if (subscriptionDataConfig == null)
{
dataType = typeof(TradeBar);
tickType = LeanData.GetCommonTickTypeForCommonDataTypes(dataType, security.Type);
}
else
{
// if subscription resolution is Tick, we also need to update the data type from Tick to TradeBar/QuoteBar
if (subscriptionDataConfig.Resolution == Resolution.Tick)
{
dataType = LeanData.GetDataType(resolution, subscriptionDataConfig.TickType);
subscriptionDataConfig = new SubscriptionDataConfig(subscriptionDataConfig, dataType, resolution: resolution);
}
dataType = subscriptionDataConfig.Type;
tickType = subscriptionDataConfig.TickType;
}
Func <int, BaseData> getLastKnownPriceForPeriods = backwardsPeriods =>
{
var startTimeUtc = _historyRequestFactory
.GetStartTimeAlgoTz(security.Symbol, backwardsPeriods, resolution, security.Exchange.Hours, dataTimeZone)
.ConvertToUtc(_localTimeKeeper.TimeZone);
var request = new HistoryRequest(
startTimeUtc,
UtcTime,
dataType,
security.Symbol,
resolution,
security.Exchange.Hours,
dataTimeZone,
resolution,
isExtendedMarketHours,
configs.IsCustomData(),
configs.DataNormalizationMode(),
tickType
);
BaseData result = null;
History(new List <HistoryRequest> {
request
})
.PushThrough(bar =>
{
if (!bar.IsFillForward)
{
result = bar;
}
});
return(result);
};
var lastKnownPrice = getLastKnownPriceForPeriods(1);
if (lastKnownPrice != null)
{
return(lastKnownPrice);
}
// If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
// We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
var periods =
resolution == Resolution.Daily ? 3 :
resolution == Resolution.Hour ? 24 : 1440;
return(getLastKnownPriceForPeriods(periods));
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
return(ParseEquity(config, line, date));
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var path = Path.Combine(Globals.DataFolder, "equity", config.Market, "fundamental", "coarse", Invariant($"{date:yyyyMMdd}.csv"));
return(new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile, FileFormat.Csv));
}
/// <summary>
/// Returns a SubscriptionDataSource for the FineFundamental class,
/// returning data from a previous date if not available for the requested date
/// </summary>
private SubscriptionDataSource GetSource(FineFundamental fine, SubscriptionDataConfig config, DateTime date)
{
var source = fine.GetSource(config, date, false);
var fileName = date.ToString("yyyyMMdd");
if (!File.Exists(source.Source))
{
if (_lastUsedFileName == null)
{
// find first file date
var path = Path.GetDirectoryName(source.Source) ?? string.Empty;
if (string.IsNullOrEmpty(path) || !Directory.Exists(path))
{
return(source);
}
var firstFileName = Path.GetFileNameWithoutExtension(Directory.GetFiles(path, "*.zip").OrderBy(x => x).First());
var firstDate = DateTime.ParseExact(firstFileName, "yyyyMMdd", CultureInfo.InvariantCulture);
// requested date before first date, return current invalid source anyway
if (date < firstDate)
{
return(source);
}
// requested date after first date, save date of first existing file
_lastUsedFileName = firstFileName;
// loop back in time until we find an existing file
while (string.CompareOrdinal(fileName, _lastUsedFileName) > 0)
{
// get previous date
date = date.AddDays(-1);
// get file name for this date
source = fine.GetSource(config, date, false);
fileName = Path.GetFileNameWithoutExtension(source.Source);
if (!File.Exists(source.Source))
{
continue;
}
// we found the file, save its name and return the source
_lastUsedFileName = fileName;
break;
}
}
else
{
// return source for last existing file date
date = DateTime.ParseExact(_lastUsedFileName, "yyyyMMdd", CultureInfo.InvariantCulture);
source = fine.GetSource(config, date, false);
}
}
else
{
_lastUsedFileName = fileName;
}
return(source);
}
/// <summary>
/// Parse an open interest data line from quantconnect zip source files.
/// </summary>
/// <param name="line">CSV source line of the compressed source</param>
/// <param name="date">Base date for the open interest (date is stored as int milliseconds since midnight)</param>
/// <param name="config">Subscription configuration object</param>
public OpenInterest(SubscriptionDataConfig config, string line, DateTime date) :
this(config, config.Symbol, line, date)
{
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
throw new NotSupportedException("RenkoBar does not support the GetSource function. This function should never be called on this type.");
}
/// <summary>
/// Creates a new instance of the <see cref="ManualUniverse"/>
/// </summary>
public ManualUniverse(SubscriptionDataConfig configuration,
UniverseSettings universeSettings,
Symbol[] symbols)
: base(configuration, universeSettings, Time.MaxTimeSpan, symbols)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionRequest"/> class
/// </summary>
public SubscriptionRequest(bool isUniverseSubscription,
Universe universe,
Security security,
SubscriptionDataConfig configuration,
DateTime startTimeUtc,
DateTime endTimeUtc)
{
IsUniverseSubscription = isUniverseSubscription;
Universe = universe;
Security = security;
Configuration = configuration;
StartTimeUtc = startTimeUtc;
EndTimeUtc = endTimeUtc;
_localStartTime = new Lazy<DateTime>(() => StartTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
_localEndTime = new Lazy<DateTime>(() => EndTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
ReaderWasCalled = true;
return(base.Reader(config, line, date, isLiveMode));
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// TradeBar Reader: Fetch the data from the QC storage and feed it line by line into the engine.
/// </summary>
/// <param name="datafeed">Destination for the this datafeed - live or backtesting</param>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">Date of this reader request</param>
/// <returns>Enumerable iterator for returning each line of the required data.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//Initialize:
var tradeBar = new TradeBar();
//Handle end of file:
if (line == null)
{
return null;
}
//Select the URL source of the data depending on where the system is trading.
switch (datafeed)
{
//Amazon S3 Backtesting Data:
case DataFeedEndpoint.Backtesting:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//Localhost Data Source
case DataFeedEndpoint.FileSystem:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//QuantConnect Live Tick Stream:
case DataFeedEndpoint.LiveTrading:
break;
}
//Return initialized TradeBar:
return tradeBar;
}
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
return(new SubscriptionDataSource("NonExistingFile", SubscriptionTransportMedium.LocalFile));
}
/// <summary>
/// Parse a tick data line from quantconnect zip source files.
/// </summary>
/// <param name="line">CSV source line of the compressed source</param>
/// <param name="date">Base date for the tick (ticks date is stored as int milliseconds since midnight)</param>
/// <param name="config">Subscription configuration object</param>
public Tick(SubscriptionDataConfig config, string line, DateTime date)
{
try
{
var csv = line.Split(',');
DataType = MarketDataType.Tick;
// Which security type is this data feed:
switch (config.SecurityType)
{
case SecurityType.Equity:
Symbol = config.Symbol;
Time = date.Date.AddMilliseconds(csv[0].ToInt64());
Value = config.GetNormalizedPrice(csv[1].ToDecimal() / 10000m);
TickType = TickType.Trade;
Quantity = csv[2].ToInt32();
if (csv.Length > 3)
{
Exchange = csv[3];
SaleCondition = csv[4];
Suspicious = (csv[5] == "1");
}
break;
case SecurityType.Forex:
Symbol = config.Symbol;
TickType = TickType.Quote;
Time = date.Date.AddMilliseconds(csv[0].ToInt64());
BidPrice = csv[1].ToDecimal();
AskPrice = csv[2].ToDecimal();
Value = (BidPrice + AskPrice) / 2;
break;
}
}
catch (Exception err)
{
Log.Error("Error Generating Tick: " + err.Message);
}
}
/// <summary>
/// Removes the specified configuration
/// </summary>
/// <param name="dataConfig">Subscription config to be removed</param>
public void Unsubscribe(SubscriptionDataConfig dataConfig)
{
SubscriptionManager.Unsubscribe(dataConfig);
_aggregator.Remove(dataConfig);
}
/// <summary>
/// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="date">Date of the data file we're looking for</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>STRING API Url for Quandl.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var source = @"https://www.quandl.com/api/v3/datasets/" + config.Symbol.Value + ".csv?order=asc&api_key=" + _authCode;
return(new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile));
}
/// <summary>
/// Convenience method for retrieving exchange hours from market hours database using a subscription config
/// </summary>
/// <param name="configuration">The subscription data config to get exchange hours for</param>
/// <returns>The configure exchange hours for the specified configuration</returns>
public SecurityExchangeHours GetExchangeHours(SubscriptionDataConfig configuration)
{
return(GetExchangeHours(configuration.Market, configuration.Symbol, configuration.SecurityType));
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
if (_random.NextDouble() < 0.01)
{
// this is simulating that we don't have data for this time period for fill forward testing
return null;
}
config.Security = _type;
var tradeBar = (TradeBar)base.Reader(config, line, date, datafeed);
return CreateFromTradeBar(tradeBar);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var url = isLiveMode ? LiveUrl : BacktestUrl;
return(new SubscriptionDataSource(url, SubscriptionTransportMedium.RemoteFile));
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary <Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24) },
{ Resolution.Hour, TimeSpan.FromHours(1) },
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >()
{
{ TickType.Quote, () => new QuoteBarConsolidator(timeSpan) },
{ TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) },
{ TickType.Trade, () => new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary <string, IDataConsolidator>();
var configs = new Dictionary <string, SubscriptionDataConfig>();
var outputFiles = new Dictionary <string, StringBuilder>();
var futures = new Dictionary <string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var consolidator = consolidators[future.Value];
foreach (var bar in leanDataReader.Parse())
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
{
consolidator.Scan(date);
}
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}
/// <summary>
/// This function is purposefully not implemented.
/// </summary>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotImplementedException("IndicatorDataPoint does not support the GetSource function. This function should never be called on this type.");
}
/// <summary>
/// QuoteBar Reader: Fetch the data from the QC storage and feed it line by line into the engine.
/// </summary>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">Date of this reader request</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>Enumerable iterator for returning each line of the required data.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
throw new NotImplementedException("Equity quote bars data format has not yet been finalized.");
}
public void Performance(Type streamReaderType, Type readLineReaderType, TickType tickType)
{
var streamReaderMilliSeconds = 0L;
var streamReaderCount = 0;
var getLineReaderMilliSeconds = 0L;
var getLineReaderCount = 0;
var stopWatch = new Stopwatch();
{
var config = new SubscriptionDataConfig(
streamReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = streamReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
streamReaderCount = reader.Read(source).Count();
streamReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
streamReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
streamReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
{
var config = new SubscriptionDataConfig(
readLineReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = readLineReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
getLineReaderCount = reader.Read(source).Count();
getLineReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
getLineReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
getLineReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
Console.WriteLine($"StreamReader: {streamReaderMilliSeconds}ms. Count {streamReaderCount}");
Console.WriteLine($"GetLine Reader: {getLineReaderMilliSeconds}ms. Count {getLineReaderCount}");
// its 50% faster but lets leave some room to avoid noise
Assert.IsTrue((streamReaderMilliSeconds * 1.5d) < getLineReaderMilliSeconds);
Assert.AreEqual(getLineReaderCount, streamReaderCount);
}
/// <summary>
/// Reader Method :: using set of arguements we specify read out type. Enumerate
/// until the end of the data stream or file. E.g. Read CSV file line by line and convert
/// into data types.
/// </summary>
/// <returns>BaseData type set by Subscription Method.</returns>
/// <param name="config">Config.</param>
/// <param name="line">Line.</param>
/// <param name="date">Date.</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
throw new NotSupportedException("RenkoBar does not support the Reader function. This function should never be called on this type.");
}
/// <summary>
/// Parses CFD trade bar data into the specified tradebar type, useful for custom types with OHLCV data deriving from TradeBar
/// </summary>
/// <typeparam name="T">The requested output type, must derive from TradeBar</typeparam>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">The base data used to compute the time of the bar since the line specifies a milliseconds since midnight</param>
/// <returns></returns>
public static T ParseCfd <T>(SubscriptionDataConfig config, string line, DateTime date)
where T : TradeBar, new()
{
// CFD has the same data format as Forex
return(ParseForex <T>(config, line, date));
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionRequest"/> class
/// </summary>
public SubscriptionRequest(SubscriptionRequest template,
bool? isUniverseSubscription = null,
Universe universe = null,
Security security = null,
SubscriptionDataConfig configuration = null,
DateTime? startTimeUtc = null,
DateTime? endTimeUtc = null
)
: this(isUniverseSubscription ?? template.IsUniverseSubscription,
universe ?? template.Universe,
security ?? template.Security,
configuration ?? template.Configuration,
startTimeUtc ?? template.StartTimeUtc,
endTimeUtc ?? template.EndTimeUtc
)
{
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called. The returned object is assumed to be time stamped in the config.ExchangeTimeZone.
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
return JsonConvert.DeserializeObject<FineFundamental>(line);
}
public void EnumerationWhileUpdatingDoesNotThrow()
{
var cts = new CancellationTokenSource();
var subscriptions = new SubscriptionCollection();
var start = DateTime.UtcNow;
var end = start.AddSeconds(10);
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, DateTimeZone.Utc, DateTimeZone.Utc, true, false, false);
var security = new Equity(
Symbols.SPY,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
var enumerator = new EnqueueableEnumerator <BaseData>();
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
var subscription = new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider);
var addTask = new TaskFactory().StartNew(() =>
{
Console.WriteLine("Add task started");
while (DateTime.UtcNow < end)
{
if (!subscriptions.Contains(config))
{
subscriptions.TryAdd(subscription);
}
Thread.Sleep(1);
}
Console.WriteLine("Add task ended");
}, cts.Token);
var removeTask = new TaskFactory().StartNew(() =>
{
Console.WriteLine("Remove task started");
while (DateTime.UtcNow < end)
{
Subscription removed;
subscriptions.TryRemove(config, out removed);
Thread.Sleep(1);
}
Console.WriteLine("Remove task ended");
}, cts.Token);
var readTask = new TaskFactory().StartNew(() =>
{
Console.WriteLine("Read task started");
while (DateTime.UtcNow < end)
{
foreach (var sub in subscriptions)
{
}
Thread.Sleep(1);
}
Console.WriteLine("Read task ended");
}, cts.Token);
Task.WaitAll(addTask, removeTask, readTask);
subscription.Dispose();
}
public void ReaderDeserializesInUtc()
{
var date = new DateTime(2020, 3, 19);
var spy = Symbol.Create("SPY", SecurityType.Equity, QuantConnect.Market.USA);
var converter = new BenzingaNewsJsonConverter();
var instance = new BenzingaNews
{
Author = "",
Categories = new List <string>(),
Contents = "",
CreatedAt = date,
Id = default(int),
Symbol = spy,
Symbols = new List <Symbol>
{
Symbol.Create("AAPL", SecurityType.Equity, QuantConnect.Market.USA),
Symbol.Create("SPY", SecurityType.Equity, QuantConnect.Market.USA)
},
Tags = new List <string>
{
"Politics"
},
Teaser = "",
Time = date,
Title = "",
UpdatedAt = date,
};
var serialized = JsonConvert.SerializeObject(instance, new JsonSerializerSettings
{
Converters = new[] { converter },
Formatting = Formatting.None,
DateTimeZoneHandling = DateTimeZoneHandling.Utc
});
var config = new SubscriptionDataConfig(
typeof(BenzingaNews),
spy,
Resolution.Minute,
TimeZones.Utc,
TimeZones.Utc,
true,
false,
false);
var news = new BenzingaNews();
var instanceRoundTripActual = (BenzingaNews)news.Reader(config, serialized, default(DateTime), false);
Assert.AreEqual(instance.Author, instanceRoundTripActual.Author);
Assert.IsTrue(instance.Categories.SequenceEqual(instanceRoundTripActual.Categories));
Assert.AreEqual(instance.Contents, instanceRoundTripActual.Contents);
Assert.AreEqual(instance.CreatedAt, instanceRoundTripActual.CreatedAt);
Assert.AreEqual(instance.Id, instanceRoundTripActual.Id);
Assert.AreEqual(instance.Symbol, instanceRoundTripActual.Symbol);
Assert.IsTrue(instance.Symbols.SequenceEqual(instanceRoundTripActual.Symbols));
Assert.IsTrue(instance.Tags.SequenceEqual(instanceRoundTripActual.Tags));
Assert.AreEqual(instance.Teaser, instanceRoundTripActual.Teaser);
Assert.AreEqual(instance.Time, instanceRoundTripActual.Time);
Assert.AreEqual(instance.Title, instanceRoundTripActual.Title);
Assert.AreEqual(instance.UpdatedAt, instanceRoundTripActual.UpdatedAt);
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="datafeed">Source of the datafeed</param>
/// <returns>String source location of the file</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
var source = "";
var dataType = TickType.Trade;
switch (datafeed)
{
//Backtesting S3 Endpoint:
case DataFeedEndpoint.Backtesting:
case DataFeedEndpoint.FileSystem:
var dateFormat = "yyyyMMdd";
if (config.Security == SecurityType.Forex)
{
dataType = TickType.Quote;
dateFormat = "yyMMdd";
}
source = @"../../../Data/" + config.Security.ToString().ToLower();
source += @"/" + config.Resolution.ToString().ToLower() + @"/" + config.Symbol.ToLower() + @"/";
source += date.ToString(dateFormat) + "_" + dataType.ToString().ToLower() + ".zip";
break;
//Live Trading Endpoint: Fake, not actually used but need for consistency with backtesting system. Set to "" so will not use subscription reader.
case DataFeedEndpoint.LiveTrading:
source = "";
break;
}
return source;
}
/// <summary>
/// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataCacheProvider">Used to cache data</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public ZipEntryNameSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataCacheProvider = dataCacheProvider;
_config = config;
_date = date;
_isLiveMode = isLiveMode;
_factory = _factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
}
/// <summary>
/// Initializes a new instance of the <see cref="FuncUniverse"/> class
/// </summary>
/// <param name="configuration">The configuration used to resolve the data for universe selection</param>
/// <param name="subscriptionSettings">The settings used for new subscriptions generated by this universe</param>
/// <param name="universeSelector">Returns the symbols that should be included in the universe</param>
public FuncUniverse(SubscriptionDataConfig configuration, SubscriptionSettings subscriptionSettings, Func<IEnumerable<BaseData>, IEnumerable<Symbol>> universeSelector)
: base(configuration)
{
_universeSelector = universeSelector;
_subscriptionSettings = subscriptionSettings;
}
public void RefreshesFutureChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 17, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CFE, "/VX");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CFE, canonicalSymbol, SecurityType.Future);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.Chicago,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var future = new Future(
canonicalSymbol,
exchangeHours,
quoteCurrency,
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
using var universe = new FuturesChainUniverse(future, universeSettings);
var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
var enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, timeProvider);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
timeProvider.Advance(Time.OneSecond);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneDay);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
enumerator.Dispose();
}
/// <summary>
/// Parse a tick data line from quantconnect zip source files.
/// </summary>
/// <param name="line">CSV source line of the compressed source</param>
/// <param name="date">Base date for the tick (ticks date is stored as int milliseconds since midnight)</param>
/// <param name="config">Subscription configuration object</param>
public Tick(SubscriptionDataConfig config, string line, DateTime date)
{
try
{
DataType = MarketDataType.Tick;
// Which security type is this data feed:
switch (config.SecurityType)
{
case SecurityType.Equity:
{
var csv = line.ToCsv(6);
Symbol = config.Symbol;
Time = date.Date.AddMilliseconds(csv[0].ToInt64()).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
Value = config.GetNormalizedPrice(csv[1].ToDecimal() / 10000m);
TickType = TickType.Trade;
Quantity = csv[2].ToInt32();
if (csv.Count > 3)
{
Exchange = csv[3];
SaleCondition = csv[4];
Suspicious = (csv[5] == "1");
}
break;
}
case SecurityType.Forex:
case SecurityType.Cfd:
{
var csv = line.ToCsv(3);
Symbol = config.Symbol;
TickType = TickType.Quote;
Time = date.Date.AddMilliseconds(csv[0].ToInt64()).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
BidPrice = csv[1].ToDecimal();
AskPrice = csv[2].ToDecimal();
Value = (BidPrice + AskPrice) / 2;
break;
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
private static DateRules GetDateRules()
{
var timeKeeper = new TimeKeeper(DateTime.Today, new List <DateTimeZone>());
var manager = new SecurityManager(timeKeeper);
// Add SPY for Equity testing
var securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, null, SecurityType.Equity);
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.SPY,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add BTC for Crypto testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.Bitfinex, Symbols.BTCUSD, SecurityType.Crypto);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.BTCUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.BTCUSD,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add EURUSD for Forex testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.FXCM, Symbols.EURUSD, SecurityType.Forex);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.EURUSD, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.EURUSD,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
// Add Fut_SPY_Feb19_2016 for testing
securityExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CME, Symbols.Fut_SPY_Feb19_2016, SecurityType.Future);
config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.Fut_SPY_Feb19_2016, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, false, false);
manager.Add(
Symbols.Fut_SPY_Feb19_2016,
new Security(
securityExchangeHours,
config,
new Cash(Currencies.USD, 0, 1m),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
)
);
var rules = new DateRules(manager, TimeZones.NewYork);
return(rules);
}
/// <summary>
/// Get source for tick data feed - not used with QuantConnect data sources implementation.
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String source location of the file to be opened with a stream</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var dataType = TickType.Trade;
if (isLiveMode)
{
// Currently ticks aren't sourced through GetSource in live mode
return new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile);
}
var dateFormat = "yyyyMMdd";
if (config.SecurityType == SecurityType.Forex)
{
dataType = TickType.Quote;
}
var symbol = string.IsNullOrEmpty(config.MappedSymbol) ? config.Symbol : config.MappedSymbol;
var securityType = config.SecurityType.ToString().ToLower();
var market = config.Market.ToLower();
var resolution = config.Resolution.ToString().ToLower();
var file = date.ToString(dateFormat) + "_" + dataType.ToString().ToLower() + ".zip";
//Add in the market for equities/cfd/forex for internationalization support.
var source = Path.Combine(Constants.DataFolder, securityType, market, resolution, symbol.ToLower(), file);
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="datafeed">Type of datafeed we're reqesting - backtest or live</param>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <returns>String URL of source file.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotSupportedException("RenkoBar does not support the GetSource function. This function should never be called on this type.");
}
/// <summary>
/// Parses Future trade bar data into the specified tradebar type, useful for custom types with OHLCV data deriving from TradeBar
/// </summary>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">The base data used to compute the time of the bar since the line specifies a milliseconds since midnight</param>
/// <returns></returns>
public static TradeBar ParseFuture(SubscriptionDataConfig config, string line, DateTime date)
{
return(ParseFuture <TradeBar>(config, line, date));
}
/// <summary>
/// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="date">Date of the data file we're looking for</param>
/// <param name="datafeed"></param>
/// <returns>STRING API Url for Quandl.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return(@"https://www.quandl.com/api/v1/datasets/" + config.Symbol + ".csv?sort_order=asc&exclude_headers=false&auth_token=" + _authCode);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var path = Path.Combine(Globals.DataFolder, "equity", config.Market, "fundamental", "coarse", date.ToString("yyyyMMdd") + ".csv");
return new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
}
/// <summary>
/// Adds the security to the user defined universe
/// </summary>
/// <param name="security">The security to add</param>
/// <param name="configurations">The <see cref="SubscriptionDataConfig"/> instances we want to add</param>
private void AddToUserDefinedUniverse(
Security security,
List <SubscriptionDataConfig> configurations)
{
var subscription = configurations.First();
// if we are adding a non-internal security which already has an internal feed, we remove it first
Security existingSecurity;
if (Securities.TryGetValue(security.Symbol, out existingSecurity))
{
if (!subscription.IsInternalFeed && existingSecurity.IsInternalFeed())
{
var securityUniverse = UniverseManager.Select(x => x.Value).OfType <UserDefinedUniverse>().FirstOrDefault(x => x.Members.ContainsKey(security.Symbol));
securityUniverse?.Remove(security.Symbol);
Securities.Remove(security.Symbol);
}
}
Securities.Add(security);
// add this security to the user defined universe
Universe universe;
var universeSymbol = UserDefinedUniverse.CreateSymbol(security.Type, security.Symbol.ID.Market);
lock (_pendingUniverseAdditionsLock)
{
if (!UniverseManager.TryGetValue(universeSymbol, out universe))
{
universe = _pendingUniverseAdditions.FirstOrDefault(x => x.Configuration.Symbol == universeSymbol);
if (universe == null)
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var uconfig = new SubscriptionDataConfig(subscription, symbol: universeSymbol, isInternalFeed: true, fillForward: false);
if (security.Type == SecurityType.Base)
{
// set entry in market hours database for the universe subscription to match the custom data
var symbolString = MarketHoursDatabase.GetDatabaseSymbolKey(uconfig.Symbol);
MarketHoursDatabase.SetEntry(uconfig.Market, symbolString, uconfig.SecurityType, security.Exchange.Hours, uconfig.DataTimeZone);
}
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(
subscription.Resolution,
security.Leverage,
subscription.FillDataForward,
subscription.ExtendedMarketHours,
TimeSpan.Zero),
QuantConnect.Time.MaxTimeSpan,
new List <Symbol>());
_pendingUniverseAdditions.Add(universe);
}
}
}
var userDefinedUniverse = universe as UserDefinedUniverse;
if (userDefinedUniverse != null)
{
lock (_pendingUniverseAdditionsLock)
{
_pendingUserDefinedUniverseSecurityAdditions.Add(
new UserDefinedUniverseAddition(userDefinedUniverse, configurations, security));
}
}
else
{
// should never happen, someone would need to add a non-user defined universe with this symbol
throw new Exception("Expected universe with symbol '" + universeSymbol.Value + "' to be of type UserDefinedUniverse.");
}
}
/// <summary>
/// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="date">Date of the data file we're looking for</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>STRING API Url for Quandl.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var source = @"https://www.quandl.com/api/v1/datasets/" + config.Symbol + ".csv?sort_order=asc&exclude_headers=false&auth_token=" + _authCode;
return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
// this data is derived from map files and factor files in backtesting
throw new NotImplementedException("This method is not supposed to be called on the Split type.");
}
