/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new SubscriptionCollection(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var config = universe.Configuration; var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
public void Initialize( AlgorithmNodePacket job, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Action <int> statusUpdate ) { throw new NotImplementedException(); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">The factor file provider to use</param> /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param> /// <param name="startTime">Start date for the data request</param> public virtual void Initialize( SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, DateTime startTime) { _config = config; var mapFile = mapFileProvider.ResolveMapFile(_config); DelistingDate = new ReferenceWrapper <DateTime>(config.Symbol.GetDelistingDate(mapFile)); }
/// <summary> /// Initializes a new instance of the <see cref="CoarseUniverseGeneratorProgram"/> class. /// </summary> /// <param name="dailyDataFolder">The daily data folder.</param> /// <param name="destinationFolder">The destination folder.</param> /// <param name="market">The market.</param> /// <param name="blackListedTickersFile">The black listed tickers file.</param> /// <param name="reservedWordsPrefix">The reserved words prefix.</param> /// <param name="mapFileProvider">The map file provider.</param> /// <param name="factorFileProvider">The factor file provider.</param> /// <param name="debugEnabled">if set to <c>true</c> [debug enabled].</param> public CoarseUniverseGeneratorProgram(DirectoryInfo dailyDataFolder, DirectoryInfo destinationFolder, string market, FileInfo blackListedTickersFile, string reservedWordsPrefix, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, bool debugEnabled = false) { _blackListedTickersFile = blackListedTickersFile; _market = market; _factorFileProvider = factorFileProvider; _mapFileProvider = mapFileProvider; _destinationFolder = destinationFolder; _dailyDataFolder = dailyDataFolder; Log.DebuggingEnabled = debugEnabled; }
/// <summary> /// Creates a new instance of the <see cref="PriceScaleFactorEnumerator"/>. /// </summary> /// <param name="rawDataEnumerator">The underlying raw data enumerator</param> /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for. /// Will determine the <see cref="DataNormalizationMode"/> to use.</param> /// <param name="factorFileProvider">The <see cref="IFactorFileProvider"/> instance to use</param> /// <param name="liveMode">True, is this is a live mode data stream</param> public PriceScaleFactorEnumerator( IEnumerator <BaseData> rawDataEnumerator, SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, bool liveMode = false) { _config = config; _liveMode = liveMode; _nextTradableDate = DateTime.MinValue; _rawDataEnumerator = rawDataEnumerator; _factorFileProvider = factorFileProvider; }
public void Initialize( IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider ) { }
public void Initialize( IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager ) { throw new System.NotImplementedException(); }
/// <summary> /// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the /// corporate event providers /// </summary> /// <param name="rawDataEnumerator">The underlying raw data enumerator</param> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="mapFileResolver">Used for resolving the correct map files</param> /// <param name="includeAuxiliaryData">True to emit auxiliary data</param> /// <param name="startTime">Start date for the data request</param> /// <param name="enablePriceScaling">Applies price factor</param> /// <returns>The new auxiliary data enumerator</returns> public static IEnumerator <BaseData> CreateEnumerators( IEnumerator <BaseData> rawDataEnumerator, SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, ITradableDatesNotifier tradableDayNotifier, MapFileResolver mapFileResolver, bool includeAuxiliaryData, DateTime startTime, bool enablePriceScaling = true) { var lazyFactorFile = new Lazy <FactorFile>(() => SubscriptionUtils.GetFactorFileToUse(config, factorFileProvider)); var tradableEventProviders = new List <ITradableDateEventProvider>(); if (config.Symbol.SecurityType == SecurityType.Equity) { tradableEventProviders.Add(new SplitEventProvider()); tradableEventProviders.Add(new DividendEventProvider()); } if (config.Symbol.SecurityType == SecurityType.Equity || config.Symbol.SecurityType == SecurityType.Base || config.Symbol.SecurityType == SecurityType.Option) { tradableEventProviders.Add(new MappingEventProvider()); } tradableEventProviders.Add(new DelistingEventProvider()); var enumerator = new AuxiliaryDataEnumerator( config, lazyFactorFile, new Lazy <MapFile>(() => GetMapFileToUse(config, mapFileResolver)), tradableEventProviders.ToArray(), tradableDayNotifier, includeAuxiliaryData, startTime); // avoid price scaling for backtesting; calculate it directly in worker // and allow subscription to extract the the data depending on config data mode var dataEnumerator = rawDataEnumerator; if (enablePriceScaling) { dataEnumerator = new PriceScaleFactorEnumerator( rawDataEnumerator, config, lazyFactorFile); } return(new SynchronizingEnumerator(dataEnumerator, enumerator)); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new ConcurrentDictionary <Symbol, Subscription>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneSecond; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // find the minimum resolution, ignoring ticks ffres = ResolveFillForwardResolution(algorithm); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { Subscription subscription; if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription)) { RemoveSubscription(subscription); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <inheritdoc /> public void Initialize( IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider ) { throw new NotImplementedException("Unexpected usage of null data feed implementation."); }
/// <summary> /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers /// </summary> /// <param name="results">The result handler for communicating results from the algorithm</param> /// <param name="setup">The setup handler used to initialize algorithm state</param> /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param> /// <param name="transactions">The transaction handler used to process orders from the algorithm</param> /// <param name="realTime">The real time handler used to process real time events</param> /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param> /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param> /// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param> /// <param name="dataProvider">file provider used to retrieve security data if it is not on the file system</param> public LeanEngineAlgorithmHandlers(IResultHandler results, ISetupHandler setup, IDataFeed dataFeed, ITransactionHandler transactions, IRealTimeHandler realTime, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider ) { if (results == null) { throw new ArgumentNullException("results"); } if (setup == null) { throw new ArgumentNullException("setup"); } if (dataFeed == null) { throw new ArgumentNullException("dataFeed"); } if (transactions == null) { throw new ArgumentNullException("transactions"); } if (realTime == null) { throw new ArgumentNullException("realTime"); } if (mapFileProvider == null) { throw new ArgumentNullException("mapFileProvider"); } if (factorFileProvider == null) { throw new ArgumentNullException("factorFileProvider"); } if (dataProvider == null) { throw new ArgumentNullException("dataProvider"); } _results = results; _setup = setup; _dataFeed = dataFeed; _transactions = transactions; _realTime = realTime; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; }
/// <summary> /// Creates a new instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">The factor file provider to use</param> /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param> /// <param name="tradableDateEventProviders">The tradable dates event providers</param> /// <param name="startTime">Start date for the data request</param> /// <param name="timeProvider">The time provider to use</param> /// <param name="securityCache">The security cache</param> public LiveAuxiliaryDataEnumerator(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, ITradableDateEventProvider[] tradableDateEventProviders, DateTime startTime, ITimeProvider timeProvider, SecurityCache securityCache) // tradableDayNotifier: null -> we are going to trigger the new tradables events for the base implementation : base(config, factorFileProvider, mapFileProvider, tradableDateEventProviders, tradableDayNotifier: null, startTime) { _securityCache = securityCache; _timeProvider = timeProvider; // initialize providers right away so mapping happens before we subscribe Initialize(); }
/// <summary> /// Initializes this history provider to work for the specified job /// </summary> /// <param name="parameters">The initialization parameters</param> public override void Initialize(HistoryProviderInitializeParameters parameters) { if (_initialized) { // let's make sure no one tries to change our parameters values throw new InvalidOperationException("SubscriptionDataReaderHistoryProvider can only be initialized once"); } _initialized = true; _mapFileProvider = parameters.MapFileProvider; _dataCacheProvider = parameters.DataCacheProvider; _factorFileProvider = parameters.FactorFileProvider; _dataPermissionManager = parameters.DataPermissionManager; _parallelHistoryRequestsEnabled = parameters.ParallelHistoryRequestsEnabled; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new ConcurrentDictionary<Symbol, Subscription>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneSecond; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // find the minimum resolution, ignoring ticks ffres = ResolveFillForwardResolution(algorithm); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { Subscription subscription; if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription)) { RemoveSubscription(subscription); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileProvider">The map file provider</param> /// <param name="factorFileProvider">The factor file provider</param> /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> /// <param name="enablePriceScaling">Applies price factor</param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataCacheProvider cacheProvider, bool enablePriceScaling = true ) { _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataCacheProvider = cacheProvider; _numericalPrecisionLimitedWarnings = new ConcurrentDictionary <Symbol, string>(); _startDateLimitedWarnings = new ConcurrentDictionary <Symbol, string>(); _enablePriceScaling = enablePriceScaling; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="factorFileProvider">The factory file provider</param> /// <param name="isLiveMode">True if runnig live algorithm, false otherwise</param> /// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, bool isLiveMode, bool includeAuxiliaryData, Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null ) { _resultHandler = resultHandler; _mapFileResolver = mapFileResolver; _factorFileProvider = factorFileProvider; _isLiveMode = isLiveMode; _includeAuxiliaryData = includeAuxiliaryData; _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays); }
/// <summary> /// Initializes a new instance of the <see cref="HistoryProviderInitializeParameters"/> class from the specified parameters /// </summary> /// <param name="job">The job</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <param name="dataCacheProvider">Provider used to cache history data files</param> /// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param> /// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param> /// <param name="statusUpdateAction">Function used to send status updates</param> public HistoryProviderInitializeParameters( AlgorithmNodePacket job, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Action <int> statusUpdateAction) { Job = job; DataProvider = dataProvider; DataCacheProvider = dataCacheProvider; MapFileProvider = mapFileProvider; FactorFileProvider = factorFileProvider; StatusUpdateAction = statusUpdateAction; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="factorFileProvider">The factory file provider</param> /// <param name="isLiveMode">True if runnig live algorithm, false otherwise</param> /// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, bool isLiveMode, bool includeAuxiliaryData, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null ) { _resultHandler = resultHandler; _mapFileResolver = mapFileResolver; _factorFileProvider = factorFileProvider; _isLiveMode = isLiveMode; _includeAuxiliaryData = includeAuxiliaryData; _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays); }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileProvider">The map file provider</param> /// <param name="factorFileProvider">The factor file provider</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <param name="includeAuxiliaryData">True to check for auxiliary data, false otherwise</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, bool includeAuxiliaryData, Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null ) { _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false); _isLiveMode = false; _includeAuxiliaryData = includeAuxiliaryData; _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">The factor file provider to use</param> /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param> /// <param name="startTime">Start date for the data request</param> public virtual void Initialize( SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, DateTime startTime) { _mapFileProvider = mapFileProvider; Config = config; InitializeMapFile(); if (MapFile.HasData(startTime.Date)) { // initialize mapped symbol using request start date Config.MappedSymbol = MapFile.GetMappedSymbol(startTime.Date, Config.MappedSymbol); } }
/// <summary> /// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the /// corporate event providers /// </summary> /// <param name="rawDataEnumerator">The underlying raw data enumerator</param> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param> /// <param name="startTime">Start date for the data request</param> /// <param name="enablePriceScaling">Applies price factor</param> /// <returns>The new auxiliary data enumerator</returns> public static IEnumerator <BaseData> CreateEnumerators( IEnumerator <BaseData> rawDataEnumerator, SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, ITradableDatesNotifier tradableDayNotifier, IMapFileProvider mapFileProvider, DateTime startTime, bool enablePriceScaling = true) { var tradableEventProviders = new List <ITradableDateEventProvider>(); if (config.EmitSplitsAndDividends()) { tradableEventProviders.Add(new SplitEventProvider()); tradableEventProviders.Add(new DividendEventProvider()); } if (config.TickerShouldBeMapped()) { tradableEventProviders.Add(new MappingEventProvider()); } tradableEventProviders.Add(new DelistingEventProvider()); var enumerator = new AuxiliaryDataEnumerator( config, factorFileProvider, mapFileProvider, tradableEventProviders.ToArray(), tradableDayNotifier, startTime); // avoid price scaling for backtesting; calculate it directly in worker // and allow subscription to extract the the data depending on config data mode var dataEnumerator = rawDataEnumerator; if (enablePriceScaling && config.PricesShouldBeScaled()) { dataEnumerator = new PriceScaleFactorEnumerator( rawDataEnumerator, config, factorFileProvider); } return(new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator)); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider, IDataChannelProvider dataChannelProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _job = (LiveNodePacket)job; _timeProvider = dataFeedTimeProvider.TimeProvider; _dataProvider = dataProvider; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _channelProvider = dataChannelProvider; _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider; _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; _subscriptions = subscriptionManager.DataFeedSubscriptions; _dataQueueHandler = GetDataQueueHandler(); _dataQueueHandler?.SetJob(_job); // run the custom data exchange var manualEvent = new ManualResetEventSlim(false); Task.Factory.StartNew(() => { manualEvent.Set(); _customExchange.Start(_cancellationTokenSource.Token); }, TaskCreationOptions.LongRunning); manualEvent.Wait(); manualEvent.DisposeSafely(); IsActive = true; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileProvider">The map file provider</param> /// <param name="factorFileProvider">The factor file provider</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> /// <param name="enablePriceScaling">Applies price factor</param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null, bool enablePriceScaling = true ) { _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _numericalPrecisionMessageSent = new ConcurrentSet <Symbol>(); _zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false); _isLiveMode = false; _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays); _enablePriceScaling = enablePriceScaling; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _timeProvider = dataFeedTimeProvider.TimeProvider; _dataQueueHandler = GetDataQueueHandler(); _dataProvider = dataProvider; _channelProvider = GetDataChannelProvider(); _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider; _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = subscriptionManager.DataFeedSubscriptions; _universeSelection = subscriptionManager.UniverseSelection; // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); IsActive = true; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="resultHandler">The result handler for the algorithm</param> /// <param name="mapFileProvider">The map file provider</param> /// <param name="factorFileProvider">The factor file provider</param> /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param> /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator. /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param> /// <param name="enablePriceScaling">Applies price factor</param> public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataCacheProvider cacheProvider, Func <SubscriptionRequest, IEnumerable <DateTime> > tradableDaysProvider = null, bool enablePriceScaling = true ) { _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataCacheProvider = cacheProvider; _numericalPrecisionLimitedWarnings = new ConcurrentDictionary <Symbol, string>(); _startDateLimitedWarnings = new ConcurrentDictionary <Symbol, string>(); _isLiveMode = false; _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays); _enablePriceScaling = enablePriceScaling; }
/// <summary> /// Initializes a new instance of the <see cref="HistoryProviderInitializeParameters"/> class from the specified parameters /// </summary> /// <param name="job">The job</param> /// <param name="api">The API instance</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <param name="dataCacheProvider">Provider used to cache history data files</param> /// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param> /// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param> /// <param name="statusUpdateAction">Function used to send status updates</param> /// <param name="parallelHistoryRequestsEnabled">True if parallel history requests are enabled</param> public HistoryProviderInitializeParameters( AlgorithmNodePacket job, IApi api, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Action <int> statusUpdateAction, bool parallelHistoryRequestsEnabled) { Job = job; Api = api; DataProvider = dataProvider; DataCacheProvider = dataCacheProvider; MapFileProvider = mapFileProvider; FactorFileProvider = factorFileProvider; StatusUpdateAction = statusUpdateAction; ParallelHistoryRequestsEnabled = parallelHistoryRequestsEnabled; }
/// <summary> /// Creates a new instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">The factor file provider to use</param> /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param> /// <param name="tradableDateEventProviders">The tradable dates event providers</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="startTime">Start date for the data request</param> public AuxiliaryDataEnumerator( SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, ITradableDateEventProvider [] tradableDateEventProviders, ITradableDatesNotifier tradableDayNotifier, DateTime startTime) { Config = config; _startTime = startTime; _mapFileProvider = mapFileProvider; _auxiliaryData = new Queue <BaseData>(); _factorFileProvider = factorFileProvider; _tradableDateEventProviders = tradableDateEventProviders; if (tradableDayNotifier != null) { tradableDayNotifier.NewTradableDate += NewTradableDate; } }
public void SetUp() { _security = new Security(Symbols.SPY, SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), new Cash(Currencies.USD, 0, 0), SymbolProperties.GetDefault(Currencies.USD), new IdentityCurrencyConverter(Currencies.USD), RegisteredSecurityDataTypesProvider.Null, new SecurityCache()); _config = new SubscriptionDataConfig( typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false); _factorFileProvider = TestGlobals.FactorFileProvider; }
/// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="dataRequest">The data request</param> /// <param name="mapFileProvider">Used for resolving the correct map files</param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="dataCacheProvider">Used for caching files</param> /// <param name="dataProvider">The data provider to use</param> public SubscriptionDataReader(SubscriptionDataConfig config, BaseDataRequest dataRequest, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataCacheProvider dataCacheProvider, IDataProvider dataProvider) { //Save configuration of data-subscription: _config = config; //Save Start and End Dates: _periodStart = dataRequest.StartTimeLocal; _periodFinish = dataRequest.EndTimeLocal; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataCacheProvider = dataCacheProvider; //Save access to securities _tradeableDates = dataRequest.TradableDays.GetEnumerator(); _dataProvider = dataProvider; }
/// <summary> /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results. /// </summary> /// <param name="config">Subscription configuration object</param> /// <param name="periodStart">Start date for the data request/backtest</param> /// <param name="periodFinish">Finish date for the data request/backtest</param> /// <param name="mapFileResolver">Used for resolving the correct map files</param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="dataCacheProvider">Used for caching files</param> /// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's data time zone</param> /// <param name="isLiveMode">True if we're in live mode, false otherwise</param> public SubscriptionDataReader(SubscriptionDataConfig config, DateTime periodStart, DateTime periodFinish, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, IEnumerable <DateTime> tradeableDates, bool isLiveMode, IDataCacheProvider dataCacheProvider) { //Save configuration of data-subscription: _config = config; //Save Start and End Dates: _periodStart = periodStart; _periodFinish = periodFinish; _mapFileResolver = mapFileResolver; _factorFileProvider = factorFileProvider; _dataCacheProvider = dataCacheProvider; //Save access to securities _isLiveMode = isLiveMode; _tradeableDates = tradeableDates.GetEnumerator(); }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider) { _frontierUtc = algorithm.StartDate.ConvertToUtc(algorithm.TimeZone); _endTimeUtc = algorithm.EndDate.ConvertToUtc(algorithm.TimeZone); foreach (var kvp in algorithm.Securities) { var security = kvp.Value; var tick = new Tick { Symbol = security.Symbol, EndTime = _frontierUtc.ConvertFromUtc(security.Exchange.TimeZone) }; _data.Add(tick); securitiesUpdateData.Add(new UpdateData <Security>(security, typeof(Tick), new BaseData[] { tick })); _consolidatorUpdateData.Add(new UpdateData <SubscriptionDataConfig>(security.Subscriptions.First(), typeof(Tick), new BaseData[] { tick })); } _timeSlices.AddRange(GetTimeSlices().Take(int.MaxValue / 1000)); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public override void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider, IDataChannelProvider dataChannelProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _algorithm = algorithm; _job = (LiveNodePacket)job; _timeProvider = dataFeedTimeProvider.TimeProvider; _dataProvider = dataProvider; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _channelProvider = dataChannelProvider; _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider; _customExchange = GetBaseDataExchange(); _subscriptions = subscriptionManager.DataFeedSubscriptions; _dataQueueHandler = GetDataQueueHandler(); _dataQueueHandler?.SetJob(_job); // run the custom data exchange _customExchange.Start(); IsActive = true; base.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, subscriptionManager, dataFeedTimeProvider, dataChannelProvider); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _dataFileProvider = dataFileProvider; _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new SubscriptionCollection(); _bridge = new BusyBlockingCollection <TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); // Not sure if this is needed but left here because of this: // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678 UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket) job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10}; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0}; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new ConcurrentDictionary<Symbol, List<Subscription>>(); _bridge = new BusyBlockingCollection<TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { _subscriptions.Add(universe.Configuration.Symbol, CreateUniverseSubscription(universe, start, Time.EndOfTime)); UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration.Symbol); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers /// </summary> /// <param name="results">The result handler for communicating results from the algorithm</param> /// <param name="setup">The setup handler used to initialize algorithm state</param> /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param> /// <param name="transactions">The transaction handler used to process orders from the algorithm</param> /// <param name="realTime">The real time handler used to process real time events</param> /// <param name="historyProvider">The history provider used to process historical data requests</param> /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param> /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param> public LeanEngineAlgorithmHandlers(IResultHandler results, ISetupHandler setup, IDataFeed dataFeed, ITransactionHandler transactions, IRealTimeHandler realTime, IHistoryProvider historyProvider, ICommandQueueHandler commandQueue, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider ) { if (results == null) { throw new ArgumentNullException("results"); } if (setup == null) { throw new ArgumentNullException("setup"); } if (dataFeed == null) { throw new ArgumentNullException("dataFeed"); } if (transactions == null) { throw new ArgumentNullException("transactions"); } if (realTime == null) { throw new ArgumentNullException("realTime"); } if (historyProvider == null) { throw new ArgumentNullException("realTime"); } if (commandQueue == null) { throw new ArgumentNullException("commandQueue"); } if (mapFileProvider == null) { throw new ArgumentNullException("mapFileProvider"); } if (factorFileProvider == null) { throw new ArgumentNullException("factorFileProvider"); } _results = results; _setup = setup; _dataFeed = dataFeed; _transactions = transactions; _realTime = realTime; _historyProvider = historyProvider; _commandQueue = commandQueue; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket) job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10}; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0}; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new SubscriptionCollection(); _bridge = new BusyBlockingCollection<TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); // Not sure if this is needed but left here because of this: // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678 UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes this history provider to work for the specified job /// </summary> /// <param name="job">The job</param> /// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param> /// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param> /// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param> /// <param name="statusUpdate">Function used to send status updates</param> public void Initialize(AlgorithmNodePacket job, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider, Action<int> statusUpdate) { return; }