/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
return (
from date in tradableDays
let fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date)
let fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataFileProvider, fineFundamentalConfiguration, date, false)
let fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault()
select new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios()
}
).GetEnumerator();
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
return((
from date in tradableDays
let fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date)
let fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataFileProvider, fineFundamentalConfiguration, date, false)
let fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault()
select new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios()
}
).GetEnumerator());
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
return(_universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = request.Configuration.Symbol
}).GetEnumerator());
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
_subscriptions = new SubscriptionCollection();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataFileProvider, false, true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public Fallout4PluginListProvider(IDataFileProvider dataFileProvider, string defaultPluginFilePath)
{
using (var reader = new StreamReader(dataFileProvider.GetPluginListFile(defaultPluginFilePath).Open()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
line = line.Trim();
if (line.StartsWith("#") || line.Length == 0)
{
continue;
}
// In Fallout4 since 1.5 version
// active plugin file name lines start with an asterisk
if (line.StartsWith("*") && line.Length > 1)
{
plugins.Add(new PluginListEntry()
{
Filename = line.Substring(1)
});
}
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
if (_isLiveMode)
{
var localTime = request.StartTimeUtc.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
// loading the list of option contract and converting them into zip entries
var symbols = _symbolUniverse.LookupSymbols(request.Security.Symbol.Underlying.ToString(), request.Security.Type);
var zipEntries = symbols.Select(x => new ZipEntryName {
Time = localTime, Symbol = x
} as BaseData).ToList();
// creating trade bar builder enumerator to model underlying price change
var underlyingEnumerator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
// configuring the enumerator
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest, underlyingEnumerator);
return(new DataQueueOptionChainUniverseDataCollectionEnumerator(request.Security.Symbol, configuredEnumerator, zipEntries));
}
else
{
var factory = new BaseDataSubscriptionEnumeratorFactory(_mapFileResolver, _factorFileProvider);
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataFileProvider)));
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionEnumerator(sync, request.Security.Symbol));
}
}
public static DataContext CreateContext(IDataFileProvider dataFileProvider)
{
// Look for classes that extend DataContext
var types = Assembly.GetExecutingAssembly().GetTypes().Where(t => t.IsSubclassOf(typeof(DataContext)));
foreach (var type in types)
{
// having attribute DataContextAttribute
var dataContextAttribute = (DataContextAttribute)type.GetCustomAttributes(typeof(DataContextAttribute), false).FirstOrDefault();
if (dataContextAttribute != null)
{
DataContext context = (DataContext)Activator.CreateInstance(type);
// Look for plugin specified by the attribute
if (dataFileProvider.GetDataFile(FileMode.Open, dataContextAttribute.PluginFileName).Exists() && dataFileProvider.DataFolderPath.Contains(context.GetGameInstallPath()))
{
// Pass data provider to the context
context.DataFileProvider = dataFileProvider;
return(context);
}
}
}
throw new InvalidDataException("Unable to determine a suitable data context.");
}
public UnitOfWork(IDataFileProvider dataFileProvider)
{
_dataFileProvider = dataFileProvider;
BlogPosts = new BlogPostRepository(dataFileProvider);
Resume = new ResumeRepository(dataFileProvider);
InitResume();
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataFileProvider">Attempts to fetch remote file provider</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataFileProvider = dataFileProvider;
_date = date;
_config = config;
_isLiveMode = isLiveMode;
_factory = (BaseData) ObjectActivator.GetActivator(config.Type).Invoke(new object[0]);
}
/// <summary>
/// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataFileProvider">Attempts to fetch remote file</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public ZipEntryNameSubscriptionDataSourceReader(IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataFileProvider = dataFileProvider;
_config = config;
_date = date;
_isLiveMode = isLiveMode;
_factory = (BaseData)Activator.CreateInstance(config.Type);
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataFileProvider">Attempts to fetch remote file provider</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataFileProvider = dataFileProvider;
_date = date;
_config = config;
_isLiveMode = isLiveMode;
_factory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[0]);
}
/// <summary>
/// Initializes a new instance of the <see cref="ZipEntryNameSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataFileProvider">Attempts to fetch remote file</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public ZipEntryNameSubscriptionDataSourceReader(IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
_dataFileProvider = dataFileProvider;
_config = config;
_date = date;
_isLiveMode = isLiveMode;
_factory = (BaseData) Activator.CreateInstance(config.Type);
}
// Instances created by classes implementing IDataFileProvider
internal DataFile(IDataFileProvider provider, FileMode mode, string fullPath, string requestedPath)
{
this.provider = provider;
this.mode = mode;
this.requestedPath = requestedPath;
this.fullPath = fullPath;
name = Path.GetFileName(requestedPath);
}
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
/// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
/// <param name="dataFileProvider">file provider used to retrieve security data if it is not on the file system</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
ICommandQueueHandler commandQueue,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider
)
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (setup == null)
{
throw new ArgumentNullException("setup");
}
if (dataFeed == null)
{
throw new ArgumentNullException("dataFeed");
}
if (transactions == null)
{
throw new ArgumentNullException("transactions");
}
if (realTime == null)
{
throw new ArgumentNullException("realTime");
}
if (commandQueue == null)
{
throw new ArgumentNullException("commandQueue");
}
if (mapFileProvider == null)
{
throw new ArgumentNullException("mapFileProvider");
}
if (factorFileProvider == null)
{
throw new ArgumentNullException("factorFileProvider");
}
if (dataFileProvider == null)
{
throw new ArgumentNullException("dataFileProvider");
}
_results = results;
_setup = setup;
_dataFeed = dataFeed;
_transactions = transactions;
_realTime = realTime;
_commandQueue = commandQueue;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
}
// Instances created by classes implementing IDataFileProvider
internal DataFile(IDataFileProvider provider, FileMode mode, string fullPath, string requestedPath)
{
this.provider = provider;
this.mode = mode;
this.requestedPath = requestedPath;
this.fullPath = fullPath;
name = Path.GetFileName(requestedPath);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, _factory.CreateEnumerator(sr, dataFileProvider))
);
var sync = new SynchronizingEnumerator(enumerators);
return new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, _factory.CreateEnumerator(sr, dataFileProvider))
);
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol));
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var configuration = request.Configuration;
var tradableDays = _tradableDaysProvider(request);
var sourceFactory = (BaseData) Activator.CreateInstance(request.Configuration.Type);
return (
from date in tradableDays
let source = sourceFactory.GetSource(configuration, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, configuration, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate)
).GetEnumerator();
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
return(new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
_resultHandler,
_mapFileResolver,
_factorFileProvider,
_dataFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_includeAuxiliaryData
));
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
return new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
_resultHandler,
_mapFileResolver,
_factorFileProvider,
_dataFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_includeAuxiliaryData
);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
return((
from date in _tradableDaysProvider(request)
let source = sourceFactory.GetSource(request.Configuration, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, request.Configuration, date, false)
let entriesForDate = factory.Read(source)
from entry in entriesForDate
select entry
)
.GetEnumerator());
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var configuration = request.Configuration;
var tradableDays = _tradableDaysProvider(request);
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
return((
from date in tradableDays
let source = sourceFactory.GetSource(configuration, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, configuration, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate)
).GetEnumerator());
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="resultHandler">The result handler for the algorithm</param>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="factorFileProvider">The factory file provider</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <param name="isLiveMode">True if runnig live algorithm, false otherwise</param>
/// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param>
/// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
/// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider,
bool isLiveMode,
bool includeAuxiliaryData,
Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null
)
{
_resultHandler = resultHandler;
_mapFileResolver = mapFileResolver;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
_isLiveMode = isLiveMode;
_includeAuxiliaryData = includeAuxiliaryData;
_tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="resultHandler">The result handler for the algorithm</param>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="factorFileProvider">The factory file provider</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <param name="isLiveMode">True if runnig live algorithm, false otherwise</param>
/// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param>
/// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
/// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider,
bool isLiveMode,
bool includeAuxiliaryData,
Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null
)
{
_resultHandler = resultHandler;
_mapFileResolver = mapFileResolver;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
_isLiveMode = isLiveMode;
_includeAuxiliaryData = includeAuxiliaryData;
_tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataFileProvider">Retrieves files if not found on disk</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
switch (source.Format)
{
case FileFormat.Csv:
return new TextSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode);
case FileFormat.Collection:
return new CollectionSubscriptionDataSourceReader(config, date, isLiveMode);
case FileFormat.ZipEntryName:
return new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode);
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
foreach (var date in _tradableDaysProvider(request))
{
var currentSymbol = request.Configuration.MappedSymbol;
request.Configuration.MappedSymbol = GetMappedSymbol(request, date);
var source = sourceFactory.GetSource(request.Configuration, date, false);
request.Configuration.MappedSymbol = currentSymbol;
var factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, request.Configuration, date, false);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
public DefaultPluginListProvider(IDataFileProvider dataFileProvider, string defaultPluginFilePath)
{
using (var reader = new StreamReader(dataFileProvider.GetPluginListFile(defaultPluginFilePath).Open()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
line = line.Trim();
if (line.StartsWith("#") || line.Length == 0)
continue;
plugins.Add(new PluginListEntry()
{
Filename = line
});
}
}
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return(new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
_resultHandler,
mapFileResolver,
_factorFileProvider,
_dataFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_dataFileCacheProvider,
_includeAuxiliaryData
));
}
public DefaultPluginListProvider(IDataFileProvider dataFileProvider, string defaultPluginFilePath)
{
using (var reader = new StreamReader(dataFileProvider.GetPluginListFile(defaultPluginFilePath).Open()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
line = line.Trim();
if (line.StartsWith("#") || line.Length == 0)
{
continue;
}
plugins.Add(new PluginListEntry()
{
Filename = line
});
}
}
}
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
/// <param name="resultHandler">Result handler used to push error messages and perform sampling on skipped days</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="dataFileProvider">Used for getting files not present on disk</param>
/// <param name="dataFileCacheProvider">Used for caching files</param>
/// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's exchange time zone</param>
/// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
/// <param name="includeAuxilliaryData">True if we want to emit aux data, false to only emit price data</param>
public SubscriptionDataReader(SubscriptionDataConfig config,
DateTime periodStart,
DateTime periodFinish,
IResultHandler resultHandler,
MapFileResolver mapFileResolver,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider,
IEnumerable <DateTime> tradeableDates,
bool isLiveMode,
DataFileCacheProvider dataFileCacheProvider = null,
bool includeAuxilliaryData = true)
{
//Save configuration of data-subscription:
_config = config;
_auxiliaryData = new Queue <BaseData>();
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
_dataFileProvider = dataFileProvider;
_dataFileCacheProvider = dataFileCacheProvider;
//Save access to securities
_isLiveMode = isLiveMode;
_includeAuxilliaryData = includeAuxilliaryData;
//Save the type of data we'll be getting from the source.
//Create the dynamic type-activators:
var objectActivator = ObjectActivator.GetActivator(config.Type);
_resultHandler = resultHandler;
_tradeableDates = tradeableDates.GetEnumerator();
if (objectActivator == null)
{
_resultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
_factorFile = new FactorFile(config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(config.Symbol.Value, new List <MapFileRow>());
// load up the map and factor files for equities
if (!config.IsCustomData && config.SecurityType == SecurityType.Equity)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
var factorFile = factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + config.Symbol.ID + ": ");
}
}
// load up the map and factor files for underlying of equity option
if (!config.IsCustomData && config.SecurityType == SecurityType.Option)
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.Underlying.ID.Symbol, config.Symbol.Underlying.ID.Date);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "Map Factors: " + config.Symbol.ID + ": ");
}
}
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
}
/// <summary>
/// Invokes the configuration following enumerator creation
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
return _configurator(_factory.CreateEnumerator(request, dataFileProvider));
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="job">The job</param>
/// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param>
/// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <param name="statusUpdate">Function used to send status updates</param>
public void Initialize(AlgorithmNodePacket job, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider, Action <int> statusUpdate)
{
Connect();
}
public ResumeRepository(IDataFileProvider dataFileProvider) : base(dataFileProvider)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
/// </summary>
/// <param name="dataFileProvider">Attempts to fetch remote file provider</param>
/// <param name="config">The subscription's configuration</param>
/// <param name="date">The date this factory was produced to read data for</param>
/// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
public TextSubscriptionDataSourceReader(IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
: this(dataFileProvider, null, config, date, isLiveMode)
{
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataFileProvider">Retrieves files if not found on disk</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
return(ForSource(source, dataFileProvider, null, config, date, isLiveMode));
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataFileProvider">Retrieves files if not found on disk</param>
/// <param name="dataFileCacheProvider">Caches files</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataFileProvider dataFileProvider, DataFileCacheProvider dataFileCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
switch (source.Format)
{
case FileFormat.Csv:
return(new TextSubscriptionDataSourceReader(dataFileProvider, dataFileCacheProvider, config, date, isLiveMode));
case FileFormat.Collection:
return(new CollectionSubscriptionDataSourceReader(config, date, isLiveMode));
case FileFormat.ZipEntryName:
return(new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode));
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="job">The job</param>
/// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param>
/// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <param name="statusUpdate">Function used to send status updates</param>
public void Initialize(AlgorithmNodePacket job, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider, Action <int> statusUpdate)
{
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
}
public BlogPostRepository(IDataFileProvider dataFileProvider) : base(dataFileProvider)
{
}
public BaseRepository(IDataFileProvider dataFileProvider)
{
_dataFileProvider = dataFileProvider;
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="job">The job</param>
/// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param>
/// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <param name="statusUpdate">Function used to send status updates</param>
public void Initialize(AlgorithmNodePacket job, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider, Action<int> statusUpdate)
{
return;
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
return _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
.Select(x => new Tick {Time = x, Symbol = request.Configuration.Symbol}).GetEnumerator();
}
public LocalizedStringFinder(IConfigurationProvider configProvider,
IDataFileProvider dataFileProvider)
{
_configProvider = configProvider;
_dataFileProvider = dataFileProvider;
}
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
/// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
/// <param name="dataFileProvider">file provider used to retrieve security data if it is not on the file system</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
ICommandQueueHandler commandQueue,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider
)
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (setup == null)
{
throw new ArgumentNullException("setup");
}
if (dataFeed == null)
{
throw new ArgumentNullException("dataFeed");
}
if (transactions == null)
{
throw new ArgumentNullException("transactions");
}
if (realTime == null)
{
throw new ArgumentNullException("realTime");
}
if (commandQueue == null)
{
throw new ArgumentNullException("commandQueue");
}
if (mapFileProvider == null)
{
throw new ArgumentNullException("mapFileProvider");
}
if (factorFileProvider == null)
{
throw new ArgumentNullException("factorFileProvider");
}
if (dataFileProvider == null)
{
throw new ArgumentNullException("dataFileProvider");
}
_results = results;
_setup = setup;
_dataFeed = dataFeed;
_transactions = transactions;
_realTime = realTime;
_commandQueue = commandQueue;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataFileProvider = dataFileProvider;
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket) job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataFileProvider = dataFileProvider;
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection<TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
