/// <summary>
 /// Initializes this history provider to work for the specified job
 /// </summary>
 /// <param name="job">The job</param>
 /// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param>
 /// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param>
 /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
 /// <param name="statusUpdate">Function used to send status updates</param>
 /// <param name="dataCacheProvider">Provider used to cache history data files</param>
 public override void Initialize(AlgorithmNodePacket job, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Action <int> statusUpdate)
 {
     _mapFileProvider    = mapFileProvider;
     _factorFileProvider = factorFileProvider;
     _dataProvider       = dataProvider;
     _dataCacheProvider  = dataCacheProvider;
 }
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider)
        {
            if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universes.IsNullOrEmpty())
            {
                throw new Exception("No subscriptions registered and no universe defined.");
            }

            _algorithm = algorithm;
            _resultHandler = resultHandler;
            _mapFileProvider = mapFileProvider;
            _subscriptions = new ConcurrentDictionary<Symbol, Subscription>();
            _cancellationTokenSource = new CancellationTokenSource();

            IsActive = true;
            Bridge = new BusyBlockingCollection<TimeSlice>(100);

            var ffres = Time.OneSecond;
            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);

            // find the minimum resolution, ignoring ticks
            ffres = ResolveFillForwardResolution(algorithm);

            // add each universe selection subscription to the feed
            foreach (var universe in _algorithm.Universes)
            {
                var startTimeUtc = _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
                var endTimeUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
                AddUniverseSubscription(universe, startTimeUtc, endTimeUtc);
            }
        }
        /// <summary>
        /// Helper method to create a new instance.
        /// Knows which security types should create one and determines the appropriate delisting event provider to use
        /// </summary>
        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
                                     SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
                                     out IEnumerator <BaseData> enumerator)
        {
            enumerator = null;
            var securityType = dataConfig.SecurityType;

            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
            {
                var providers = new List <ITradableDateEventProvider>
                {
                    securityType == SecurityType.Equity
                        ? new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler)
                        : new DelistingEventProvider()
                };

                if (dataConfig.TickerShouldBeMapped())
                {
                    providers.Add(new LiveMappingEventProvider());
                }

                enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
                                                             providers.ToArray(), startTime, timeProvider, securityCache);
            }
            return(enumerator != null);
        }
Example #4
0
 public void OneTimeSetUp()
 {
     _dataProvider       = TestGlobals.DataProvider;
     _mapFileProvider    = TestGlobals.MapFileProvider;
     _factorFileProvider = TestGlobals.FactorFileProvider;
     _cacheProvider      = new ZipDataCacheProvider(TestGlobals.DataProvider);
 }
Example #5
0
        /// <summary>
        /// Helper overload that will search the mapfiles to resolve the first date. This implementation
        /// uses the configured <see cref="IMapFileProvider"/> via the <see cref="Composer.Instance"/>
        /// </summary>
        /// <param name="symbol">The symbol as it is known today</param>
        /// <param name="market">The market</param>
        /// <param name="mapSymbol">Specifies if symbol should be mapped using map file provider</param>
        /// <param name="mapFileProvider">Specifies the IMapFileProvider to use for resolving symbols, specify null to load from Composer</param>
        /// <returns>A new <see cref="SecurityIdentifier"/> representing the specified symbol today</returns>
        public static SecurityIdentifier GenerateEquity(string symbol, string market, bool mapSymbol = true, IMapFileProvider mapFileProvider = null)
        {
            if (mapSymbol)
            {
                MapFile mapFile;
                if (mapFileProvider == null)
                {
                    lock (_mapFileProviderLock)
                    {
                        if (_mapFileProvider == null)
                        {
                            _mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(MapFileProviderTypeName);
                        }

                        mapFile = GetMapFile(_mapFileProvider, market, symbol);
                    }
                }
                else
                {
                    mapFile = GetMapFile(mapFileProvider, market, symbol);
                }

                var firstDate = mapFile.FirstDate;
                if (mapFile.Any())
                {
                    symbol = mapFile.FirstTicker;
                }

                return(GenerateEquity(firstDate, symbol, market));
            }
            else
            {
                return(GenerateEquity(DefaultDate, symbol, market));
            }
        }
Example #6
0
            public void Initialize(IAlgorithm algorithm,
                                   AlgorithmNodePacket job,
                                   IResultHandler resultHandler,
                                   IMapFileProvider mapFileProvider,
                                   IFactorFileProvider factorFileProvider,
                                   IDataProvider dataProvider,
                                   IDataFeedSubscriptionManager subscriptionManager)
            {
                _frontierUtc = algorithm.StartDate.ConvertToUtc(algorithm.TimeZone);
                _endTimeUtc  = algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
                foreach (var kvp in algorithm.Securities)
                {
                    var security = kvp.Value;
                    var tick     = new Tick
                    {
                        Symbol  = security.Symbol,
                        EndTime = _frontierUtc.ConvertFromUtc(security.Exchange.TimeZone)
                    };
                    _data.Add(tick);
                    securitiesUpdateData.Add(new UpdateData <Security>(security, typeof(Tick), new BaseData[] { tick }));
                    _consolidatorUpdateData.Add(new UpdateData <SubscriptionDataConfig>(security.Subscriptions.First(), typeof(Tick), new BaseData[] { tick }));
                }

                _timeSlices.AddRange(GetTimeSlices().Take(int.MaxValue / 1000));
            }
Example #7
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm,
                               AlgorithmNodePacket job,
                               IResultHandler resultHandler,
                               IMapFileProvider mapFileProvider,
                               IFactorFileProvider factorFileProvider,
                               IDataProvider dataProvider,
                               IDataFeedSubscriptionManager subscriptionManager,
                               IDataFeedTimeProvider dataFeedTimeProvider,
                               IDataChannelProvider dataChannelProvider)
        {
            _algorithm               = algorithm;
            _resultHandler           = resultHandler;
            _mapFileProvider         = mapFileProvider;
            _factorFileProvider      = factorFileProvider;
            _dataProvider            = dataProvider;
            _timeProvider            = dataFeedTimeProvider.FrontierTimeProvider;
            _subscriptions           = subscriptionManager.DataFeedSubscriptions;
            _cancellationTokenSource = new CancellationTokenSource();
            _subscriptionFactory     = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
                _resultHandler,
                _mapFileProvider,
                _factorFileProvider,
                _dataProvider,
                includeAuxiliaryData: true,
                enablePriceScaling: false);

            IsActive = true;
        }
Example #8
0
        public static MapFile GetMapFile(IMapFileProvider mapFileProvider, string market, string symbol)
        {
            var resolver = mapFileProvider.Get(market);
            var mapFile  = resolver.ResolveMapFile(symbol, DateTime.Today);

            return(mapFile);
        }
Example #9
0
        private static void SetupDependencies()
        {
            _typeRegistry = new UnityRegistry("EOLib.IO");

            _pubProvider     = _typeRegistry.Resolve <IPubFileProvider>();
            _mapFileProvider = _typeRegistry.Resolve <IMapFileProvider>();
        }
Example #10
0
 /// <summary>
 /// Creates a new instance
 /// </summary>
 /// <param name="dataProvider">The data provider instance to use</param>
 public BacktestingOptionChainProvider(IDataProvider dataProvider)
 {
     _dataProvider    = dataProvider;
     _mapFileProvider =
         Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider",
                                                                                    "LocalDiskMapFileProvider"));
 }
Example #11
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider)
        {
            _algorithm               = algorithm;
            _resultHandler           = resultHandler;
            _mapFileProvider         = mapFileProvider;
            _factorFileProvider      = factorFileProvider;
            _dataProvider            = dataProvider;
            _subscriptions           = new SubscriptionCollection();
            _universeSelection       = new UniverseSelection(this, algorithm);
            _cancellationTokenSource = new CancellationTokenSource();
            _subscriptionfactory     = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true);

            IsActive = true;
            var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));

            _controller = new ParallelRunnerController(threadCount);
            _controller.Start(_cancellationTokenSource.Token);

            var ffres = Time.OneMinute;

            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);

            // wire ourselves up to receive notifications when universes are added/removed
            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
            {
                switch (args.Action)
                {
                case NotifyCollectionChangedAction.Add:
                    foreach (var universe in args.NewItems.OfType <Universe>())
                    {
                        var config = universe.Configuration;
                        var start  = _algorithm.UtcTime;

                        var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
                        var exchangeHours       = marketHoursDatabase.GetExchangeHours(config);

                        Security security;
                        if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
                        {
                            // create a canonical security object if it doesn't exist
                            security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
                        }

                        var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
                        AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
                    }
                    break;

                case NotifyCollectionChangedAction.Remove:
                    foreach (var universe in args.OldItems.OfType <Universe>())
                    {
                        RemoveSubscription(universe.Configuration);
                    }
                    break;

                default:
                    throw new NotImplementedException("The specified action is not implemented: " + args.Action);
                }
            };
        }
Example #12
0
        /// <summary>
        /// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
        /// </summary>
        /// <param name="config">Subscription configuration object</param>
        /// <param name="periodStart">Start date for the data request/backtest</param>
        /// <param name="periodFinish">Finish date for the data request/backtest</param>
        /// <param name="mapFileResolver">Used for resolving the correct map files</param>
        /// <param name="factorFileProvider">Used for getting factor files</param>
        /// <param name="dataCacheProvider">Used for caching files</param>
        /// <param name="tradeableDates">Defines the dates for which we'll request data, in order, in the security's data time zone</param>
        /// <param name="isLiveMode">True if we're in live mode, false otherwise</param>
        /// <param name="dataProvider">The data provider to use</param>
        public SubscriptionDataReader(SubscriptionDataConfig config,
                                      DateTime periodStart,
                                      DateTime periodFinish,
                                      IMapFileProvider mapFileProvider,
                                      IFactorFileProvider factorFileProvider,
                                      IEnumerable <DateTime> tradeableDates,
                                      bool isLiveMode,
                                      IDataCacheProvider dataCacheProvider,
                                      IDataProvider dataProvider)
        {
            //Save configuration of data-subscription:
            _config = config;

            //Save Start and End Dates:
            _periodStart        = periodStart;
            _periodFinish       = periodFinish;
            _mapFileProvider    = mapFileProvider;
            _factorFileProvider = factorFileProvider;
            _dataCacheProvider  = dataCacheProvider;

            //Save access to securities
            _isLiveMode     = isLiveMode;
            _tradeableDates = tradeableDates.GetEnumerator();
            _dataProvider   = dataProvider;
        }
Example #13
0
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider)
        {
            if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universes.IsNullOrEmpty())
            {
                throw new Exception("No subscriptions registered and no universe defined.");
            }

            _algorithm               = algorithm;
            _resultHandler           = resultHandler;
            _mapFileProvider         = mapFileProvider;
            _subscriptions           = new ConcurrentDictionary <Symbol, Subscription>();
            _cancellationTokenSource = new CancellationTokenSource();

            IsActive = true;
            Bridge   = new BusyBlockingCollection <TimeSlice>(100);

            var ffres = Time.OneSecond;

            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);

            // find the minimum resolution, ignoring ticks
            ffres = ResolveFillForwardResolution(algorithm);

            // add each universe selection subscription to the feed
            foreach (var universe in _algorithm.Universes)
            {
                var startTimeUtc = _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
                var endTimeUtc   = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
                AddUniverseSubscription(universe, startTimeUtc, endTimeUtc);
            }
        }
        /// <summary>
        /// Initializes this instance
        /// </summary>
        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
        /// <param name="factorFileProvider">The factor file provider to use</param>
        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
        /// <param name="startTime">Start date for the data request</param>
        public override void Initialize(
            SubscriptionDataConfig config,
            IFactorFileProvider factorFileProvider,
            IMapFileProvider mapFileProvider,
            DateTime startTime)
        {
            base.Initialize(config, factorFileProvider, mapFileProvider, startTime);

            _delistingEnumerator = _dataQueueHandler.Subscribe(_dataConfig, (sender, args) =>
            {
                if (_delistingEnumerator != null && _delistingEnumerator.MoveNext())
                {
                    // live trading always returns true but could be null
                    if (_delistingEnumerator.Current != null)
                    {
                        var delisting = _delistingEnumerator.Current as Delisting;
                        if (delisting != null)
                        {
                            // we set the delisting date!
                            DelistingDate = new ReferenceWrapper <DateTime>(delisting.Time);
                        }
                        else
                        {
                            Log.Error($"LiveDataBasedDelistingEventProvider(): Current is not a {nameof(Delisting)} event: {_delistingEnumerator.Current?.GetType()}");
                        }
                    }
                }
                else
                {
                    Log.Error("LiveDataBasedDelistingEventProvider(): new data available triggered with no data");
                }
            });
        }
Example #15
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm,
                               AlgorithmNodePacket job,
                               IResultHandler resultHandler,
                               IMapFileProvider mapFileProvider,
                               IFactorFileProvider factorFileProvider,
                               IDataProvider dataProvider,
                               IDataFeedSubscriptionManager subscriptionManager,
                               IDataFeedTimeProvider dataFeedTimeProvider)
        {
            _algorithm               = algorithm;
            _resultHandler           = resultHandler;
            _mapFileProvider         = mapFileProvider;
            _factorFileProvider      = factorFileProvider;
            _dataProvider            = dataProvider;
            _subscriptions           = subscriptionManager.DataFeedSubscriptions;
            _universeSelection       = subscriptionManager.UniverseSelection;
            _cancellationTokenSource = new CancellationTokenSource();
            _subscriptionFactory     = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
                _resultHandler,
                _mapFileProvider,
                _factorFileProvider,
                _dataProvider,
                includeAuxiliaryData: true);

            IsActive = true;
            var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
        }
 /// <summary>
 /// Initializes this history provider to work for the specified job
 /// </summary>
 /// <param name="parameters">The initialization parameters</param>
 public override void Initialize(HistoryProviderInitializeParameters parameters)
 {
     _mapFileProvider                = parameters.MapFileProvider;
     _factorFileProvider             = parameters.FactorFileProvider;
     _dataCacheProvider              = parameters.DataCacheProvider;
     _parallelHistoryRequestsEnabled = parameters.ParallelHistoryRequestsEnabled;
 }
Example #17
0
        /// <summary>
        /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
        /// </summary>
        /// <param name="results">The result handler for communicating results from the algorithm</param>
        /// <param name="setup">The setup handler used to initialize algorithm state</param>
        /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
        /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
        /// <param name="realTime">The real time handler used to process real time events</param>
        /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
        /// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
        /// <param name="dataProvider">file provider used to retrieve security data if it is not on the file system</param>
        /// <param name="alphas">The alpha handler used to process generated insights</param>
        public LeanEngineAlgorithmHandlers(IResultHandler results,
                                           ISetupHandler setup,
                                           IDataFeed dataFeed,
                                           ITransactionHandler transactions,
                                           IRealTimeHandler realTime,
                                           IMapFileProvider mapFileProvider,
                                           IFactorFileProvider factorFileProvider,
                                           IDataProvider dataProvider,
                                           IAlphaHandler alphas
                                           )
        {
            if (results == null)
            {
                throw new ArgumentNullException(nameof(results));
            }
            if (setup == null)
            {
                throw new ArgumentNullException(nameof(setup));
            }
            if (dataFeed == null)
            {
                throw new ArgumentNullException(nameof(dataFeed));
            }
            if (transactions == null)
            {
                throw new ArgumentNullException(nameof(transactions));
            }
            if (realTime == null)
            {
                throw new ArgumentNullException(nameof(realTime));
            }
            if (mapFileProvider == null)
            {
                throw new ArgumentNullException(nameof(mapFileProvider));
            }
            if (factorFileProvider == null)
            {
                throw new ArgumentNullException(nameof(factorFileProvider));
            }
            if (dataProvider == null)
            {
                throw new ArgumentNullException(nameof(dataProvider));
            }
            if (alphas == null)
            {
                throw new ArgumentNullException(nameof(alphas));
            }

            Results            = results;
            Setup              = setup;
            DataFeed           = dataFeed;
            Transactions       = transactions;
            RealTime           = realTime;
            MapFileProvider    = mapFileProvider;
            FactorFileProvider = factorFileProvider;
            DataProvider       = dataProvider;
            Alphas             = alphas;
        }
Example #18
0
 /// <summary>
 /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
 /// </summary>
 /// <param name="results">The result handler for communicating results from the algorithm</param>
 /// <param name="setup">The setup handler used to initialize algorithm state</param>
 /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
 /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
 /// <param name="realTime">The real time handler used to process real time events</param>
 /// <param name="historyProvider">The history provider used to process historical data requests</param>
 /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
 /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
 public LeanEngineAlgorithmHandlers(IResultHandler results,
                                    ISetupHandler setup,
                                    IDataFeed dataFeed,
                                    ITransactionHandler transactions,
                                    IRealTimeHandler realTime,
                                    IHistoryProvider historyProvider,
                                    ICommandQueueHandler commandQueue,
                                    IMapFileProvider mapFileProvider,
                                    IFactorFileProvider factorFileProvider
                                    )
 {
     if (results == null)
     {
         throw new ArgumentNullException("results");
     }
     if (setup == null)
     {
         throw new ArgumentNullException("setup");
     }
     if (dataFeed == null)
     {
         throw new ArgumentNullException("dataFeed");
     }
     if (transactions == null)
     {
         throw new ArgumentNullException("transactions");
     }
     if (realTime == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (historyProvider == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (commandQueue == null)
     {
         throw new ArgumentNullException("commandQueue");
     }
     if (mapFileProvider == null)
     {
         throw new ArgumentNullException("mapFileProvider");
     }
     if (factorFileProvider == null)
     {
         throw new ArgumentNullException("factorFileProvider");
     }
     _results            = results;
     _setup              = setup;
     _dataFeed           = dataFeed;
     _transactions       = transactions;
     _realTime           = realTime;
     _historyProvider    = historyProvider;
     _commandQueue       = commandQueue;
     _mapFileProvider    = mapFileProvider;
     _factorFileProvider = factorFileProvider;
 }
Example #19
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
        {
            _algorithm = algorithm;
            _resultHandler = resultHandler;
            _mapFileProvider = mapFileProvider;
            _factorFileProvider = factorFileProvider;
            _subscriptions = new SubscriptionCollection();
            _universeSelection = new UniverseSelection(this, algorithm, job.Controls);
            _cancellationTokenSource = new CancellationTokenSource();

            IsActive = true;
            var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
            _controller = new ParallelRunnerController(threadCount);
            _controller.Start(_cancellationTokenSource.Token);

            var ffres = Time.OneMinute;
            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);

            // wire ourselves up to receive notifications when universes are added/removed
            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
            {
                switch (args.Action)
                {
                    case NotifyCollectionChangedAction.Add:
                        foreach (var universe in args.NewItems.OfType<Universe>())
                        {
                            var config = universe.Configuration;
                            var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);

                            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
                            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

                            Security security;
                            if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
                            {
                                // create a canonical security object if it doesn't exist
                                security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
                            }

                            var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
                            AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
                        }
                        break;

                    case NotifyCollectionChangedAction.Remove:
                        foreach (var universe in args.OldItems.OfType<Universe>())
                        {
                            RemoveSubscription(universe.Configuration);
                        }
                        break;

                    default:
                        throw new NotImplementedException("The specified action is not implemented: " + args.Action);
                }
            };
        }
Example #20
0
 /// <summary>
 /// Initializes this instance
 /// </summary>
 /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
 /// <param name="factorFileProvider">The factor file provider to use</param>
 /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
 /// <param name="startTime">Start date for the data request</param>
 public void Initialize(
     SubscriptionDataConfig config,
     IFactorFileProvider factorFileProvider,
     IMapFileProvider mapFileProvider,
     DateTime startTime)
 {
     _config     = config;
     _mapFile    = mapFileProvider.ResolveMapFile(_config);
     _factorFile = factorFileProvider.Get(_config.Symbol) as CorporateFactorProvider;
 }
Example #21
0
        /// <summary>
        /// Helper method to resolve the mapping file to use.
        /// </summary>
        /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
        /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
        /// <param name="mapFileProvider">The map file provider</param>
        /// <param name="dataConfig">The configuration to fetch the map file for</param>
        /// <returns>The mapping file to use</returns>
        public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
        {
            var resolver = MapFileResolver.Empty;

            if (dataConfig.TickerShouldBeMapped())
            {
                resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol));
            }
            return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode));
        }
Example #22
0
 public HudStateActions(IStatusLabelSetter statusLabelSetter,
                        IHudControlProvider hudControlProvider,
                        ICurrentMapStateRepository currentMapStateRepository,
                        IMapFileProvider mapFileProvider)
 {
     _statusLabelSetter         = statusLabelSetter;
     _hudControlProvider        = hudControlProvider;
     _currentMapStateRepository = currentMapStateRepository;
     _mapFileProvider           = mapFileProvider;
 }
Example #23
0
        /// <summary>
        /// Resolves the first ticker/date of the security represented by <paramref name="tickerToday"/>
        /// </summary>
        /// <param name="mapFileProvider">The IMapFileProvider instance used for resolving map files</param>
        /// <param name="tickerToday">The security's ticker as it trades today</param>
        /// <param name="market">The market the security exists in</param>
        /// <param name="mappingResolveDate">The date to use to resolve the map file. Default value is <see cref="DateTime.Today"/></param>
        /// <returns>The security's first ticker/date if mapping data available, otherwise, the provided ticker and DefaultDate are returned</returns>
        private static Tuple <string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, DateTime?mappingResolveDate = null)
        {
            var resolver = mapFileProvider.Get(market);
            var mapFile  = resolver.ResolveMapFile(tickerToday, mappingResolveDate ?? DateTime.Today);

            // if we have mapping data, use the first ticker/date from there, otherwise use provided ticker and DefaultDate
            return(mapFile.Any()
                ? Tuple.Create(mapFile.FirstTicker, mapFile.FirstDate)
                : Tuple.Create(tickerToday, DefaultDate));
        }
Example #24
0
        /// <summary>
        /// Creates a new instance
        /// </summary>
        public ContinuousContractUniverse(Security security, UniverseSettings universeSettings, bool liveMode, SubscriptionDataConfig universeConfig)
            : base(universeConfig)
        {
            _security        = security;
            _liveMode        = liveMode;
            UniverseSettings = universeSettings;
            var mapFileProviderTypeName = Config.Get("map-file-provider", "LocalDiskMapFileProvider");

            _mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(mapFileProviderTypeName);
        }
Example #25
0
        public void Setup()
        {
            _algorithm = new QCAlgorithm();
            _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
            _algorithm.HistoryProvider = _testHistoryProvider = new TestHistoryProvider();

            _dataProvider       = TestGlobals.DataProvider;
            _mapFileProvider    = TestGlobals.MapFileProvider;
            _factorFileProvider = TestGlobals.FactorFileProvider;
        }
Example #26
0
 public void Initialize(
     AlgorithmNodePacket job,
     IDataProvider dataProvider,
     IDataCacheProvider dataCacheProvider,
     IMapFileProvider mapFileProvider,
     IFactorFileProvider factorFileProvider,
     Action <int> statusUpdate
     )
 {
     throw new NotImplementedException();
 }
        /// <summary>
        /// Initializes this instance
        /// </summary>
        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
        /// <param name="factorFileProvider">The factor file provider to use</param>
        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
        /// <param name="startTime">Start date for the data request</param>
        public virtual void Initialize(
            SubscriptionDataConfig config,
            IFactorFileProvider factorFileProvider,
            IMapFileProvider mapFileProvider,
            DateTime startTime)
        {
            _config = config;
            var mapFile = mapFileProvider.ResolveMapFile(_config);

            DelistingDate = new ReferenceWrapper <DateTime>(config.Symbol.GetDelistingDate(mapFile));
        }
Example #28
0
        private static void SetupDependencies()
        {
            var dependencyRegistrar = new DependencyRegistrar(_unityContainer = new UnityContainer());
            var containers          = new IDependencyContainer[] { new IODependencyContainer() };

            dependencyRegistrar.RegisterDependencies(containers);
            dependencyRegistrar.InitializeDependencies(containers.OfType <IInitializableContainer>().ToArray());

            _pubProvider     = _unityContainer.Resolve <IPubFileProvider>();
            _mapFileProvider = _unityContainer.Resolve <IMapFileProvider>();
        }
Example #29
0
        /// <summary>
        /// Initializes our FactorFileProvider by supplying our mapFileProvider
        /// and dataProvider
        /// </summary>
        /// <param name="mapFileProvider">MapFileProvider to use</param>
        /// <param name="dataProvider">DataProvider to use</param>
        public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider)
        {
            if (_mapFileProvider != null || _dataProvider != null)
            {
                return;
            }

            _mapFileProvider = mapFileProvider;
            _dataProvider    = dataProvider;
            StartExpirationTask();
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="CoarseUniverseGeneratorProgram"/> class.
 /// </summary>
 /// <param name="dailyDataFolder">The daily data folder.</param>
 /// <param name="destinationFolder">The destination folder.</param>
 /// <param name="market">The market.</param>
 /// <param name="blackListedTickersFile">The black listed tickers file.</param>
 /// <param name="reservedWordsPrefix">The reserved words prefix.</param>
 /// <param name="mapFileProvider">The map file provider.</param>
 /// <param name="factorFileProvider">The factor file provider.</param>
 /// <param name="debugEnabled">if set to <c>true</c> [debug enabled].</param>
 public CoarseUniverseGeneratorProgram(DirectoryInfo dailyDataFolder, DirectoryInfo destinationFolder, string market, FileInfo blackListedTickersFile, string reservedWordsPrefix,
                                       IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, bool debugEnabled = false)
 {
     _blackListedTickersFile = blackListedTickersFile;
     _market              = market;
     _factorFileProvider  = factorFileProvider;
     _mapFileProvider     = mapFileProvider;
     _destinationFolder   = destinationFolder;
     _dailyDataFolder     = dailyDataFolder;
     Log.DebuggingEnabled = debugEnabled;
 }
Example #31
0
        /// <summary>
        /// Creates an instance of the symbol resolver
        /// </summary>
        /// <param name="dataProvider">Data provider used to obtain symbol mappings data</param>
        /// <param name="securitiesDefinitionKey">Location to read the securities definition data from</param>
        public SecurityDefinitionSymbolResolver(IDataProvider dataProvider = null, string securitiesDefinitionKey = null)
        {
            _securitiesDefinitionKey = securitiesDefinitionKey ?? Path.Combine(Globals.DataFolder, "symbol-properties", "security-database.csv");

            _dataProvider = dataProvider ??
                            Composer.Instance.GetExportedValueByTypeName <IDataProvider>(
                Config.Get("data-provider", "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider"));

            _mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"));
            _mapFileProvider.Initialize(_dataProvider);
        }
Example #32
0
 public void Initialize(
     IAlgorithm algorithm,
     AlgorithmNodePacket job,
     IResultHandler resultHandler,
     IMapFileProvider mapFileProvider,
     IFactorFileProvider factorFileProvider,
     IDataProvider dataProvider,
     IDataFeedSubscriptionManager subscriptionManager,
     IDataFeedTimeProvider dataFeedTimeProvider
     )
 {
 }
 /// <summary>
 /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
 /// </summary>
 /// <param name="results">The result handler for communicating results from the algorithm</param>
 /// <param name="setup">The setup handler used to initialize algorithm state</param>
 /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
 /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
 /// <param name="realTime">The real time handler used to process real time events</param>
 /// <param name="historyProvider">The history provider used to process historical data requests</param>
 /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
 /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
 public LeanEngineAlgorithmHandlers(IResultHandler results,
     ISetupHandler setup,
     IDataFeed dataFeed,
     ITransactionHandler transactions,
     IRealTimeHandler realTime,
     IHistoryProvider historyProvider,
     ICommandQueueHandler commandQueue,
     IMapFileProvider mapFileProvider
     )
 {
     if (results == null)
     {
         throw new ArgumentNullException("results");
     }
     if (setup == null)
     {
         throw new ArgumentNullException("setup");
     }
     if (dataFeed == null)
     {
         throw new ArgumentNullException("dataFeed");
     }
     if (transactions == null)
     {
         throw new ArgumentNullException("transactions");
     }
     if (realTime == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (historyProvider == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (commandQueue == null)
     {
         throw new ArgumentNullException("commandQueue");
     }
     if (mapFileProvider == null)
     {
         throw new ArgumentNullException("mapFileProvider");
     }
     _results = results;
     _setup = setup;
     _dataFeed = dataFeed;
     _transactions = transactions;
     _realTime = realTime;
     _historyProvider = historyProvider;
     _commandQueue = commandQueue;
     _mapFileProvider = mapFileProvider;
 }
Example #34
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
        {
            _algorithm = algorithm;
            _resultHandler = resultHandler;
            _mapFileProvider = mapFileProvider;
            _factorFileProvider = factorFileProvider;
            _subscriptions = new ConcurrentDictionary<Symbol, Subscription>();
            _universeSelection = new UniverseSelection(this, algorithm, job.Controls);
            _cancellationTokenSource = new CancellationTokenSource();

            IsActive = true;
            var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
            _controller = new ParallelRunnerController(threadCount);
            _controller.Start(_cancellationTokenSource.Token);

            var ffres = Time.OneSecond;
            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);

            // find the minimum resolution, ignoring ticks
            ffres = ResolveFillForwardResolution(algorithm);

            // wire ourselves up to receive notifications when universes are added/removed
            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
            {
                switch (args.Action)
                {
                    case NotifyCollectionChangedAction.Add:
                        foreach (var universe in args.NewItems.OfType<Universe>())
                        {
                            var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
                            AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
                        }
                        break;

                    case NotifyCollectionChangedAction.Remove:
                        foreach (var universe in args.OldItems.OfType<Universe>())
                        {
                            Subscription subscription;
                            if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription))
                            {
                                RemoveSubscription(subscription);
                            }
                        }
                        break;

                    default:
                        throw new NotImplementedException("The specified action is not implemented: " + args.Action);
                }
            };
        }
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider)
        {
            if (!(job is LiveNodePacket))
            {
                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
            }

            if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universes.IsNullOrEmpty())
            {
                throw new Exception("No subscriptions registered and no universe defined.");
            }

            _cancellationTokenSource = new CancellationTokenSource();

            _algorithm = algorithm;
            _job = (LiveNodePacket) job;
            _resultHandler = resultHandler;
            _timeProvider = GetTimeProvider();
            _dataQueueHandler = GetDataQueueHandler();

            _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
            _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
            // sleep is controlled on this exchange via the GetNextTicksEnumerator
            _exchange = new BaseDataExchange("DataQueueExchange", GetNextTicksEnumerator()){SleepInterval = 0};
            _subscriptions = new ConcurrentDictionary<Symbol, Subscription>();

            Bridge = new BusyBlockingCollection<TimeSlice>();

            // run the exchanges
            _exchange.Start();
            _customExchange.Start();

            // this value will be modified via calls to AddSubscription/RemoveSubscription
            var ffres = Time.OneSecond;
            _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);

            ffres = ResolveFillForwardResolution(algorithm);

            // add subscriptions
            var start = _timeProvider.GetUtcNow();
            foreach (var universe in _algorithm.Universes)
            {
                var subscription = CreateUniverseSubscription(universe, start, Time.EndOfTime);
                _subscriptions[subscription.Security.Symbol] = subscription;
            }
        }
Example #36
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
        {
            if (!(job is LiveNodePacket))
            {
                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
            }

            _cancellationTokenSource = new CancellationTokenSource();

            _algorithm = algorithm;
            _job = (LiveNodePacket) job;
            _resultHandler = resultHandler;
            _timeProvider = GetTimeProvider();
            _dataQueueHandler = GetDataQueueHandler();

            _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
            _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
            // sleep is controlled on this exchange via the GetNextTicksEnumerator
            _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
            _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
            _subscriptions = new SubscriptionCollection();

            _bridge = new BusyBlockingCollection<TimeSlice>();
            _universeSelection = new UniverseSelection(this, algorithm, job.Controls);

            // run the exchanges
            Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
            Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));

            // this value will be modified via calls to AddSubscription/RemoveSubscription
            var ffres = Time.OneMinute;
            _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);

            // wire ourselves up to receive notifications when universes are added/removed
            var start = _timeProvider.GetUtcNow();
            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
            {
                switch (args.Action)
                {
                    case NotifyCollectionChangedAction.Add:
                        foreach (var universe in args.NewItems.OfType<Universe>())
                        {
                            var config = universe.Configuration;
                            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
                            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

                            Security security;
                            if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
                            {
                                // create a canonical security object
                                security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
                            }

                            AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));

                            // Not sure if this is needed but left here because of this:
                            // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
                            UpdateFillForwardResolution();
                        }
                        break;

                    case NotifyCollectionChangedAction.Remove:
                        foreach (var universe in args.OldItems.OfType<Universe>())
                        {
                            RemoveSubscription(universe.Configuration);
                        }
                        break;

                    default:
                        throw new NotImplementedException("The specified action is not implemented: " + args.Action);
                }
            };
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="LocalDiskFactorFileProvider"/> using the specified
 /// map file provider
 /// </summary>
 /// <param name="mapFileProvider">The map file provider used to resolve permticks of securities</param>
 public LocalDiskFactorFileProvider(IMapFileProvider mapFileProvider)
 {
     _mapFileProvider = mapFileProvider;
     _cache = new ConcurrentDictionary<Symbol, FactorFile>();
 }
Example #38
0
        /// <summary>
        /// Initializes the data feed for the specified job and algorithm
        /// </summary>
        public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
        {
            if (!(job is LiveNodePacket))
            {
                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
            }

            _cancellationTokenSource = new CancellationTokenSource();

            _algorithm = algorithm;
            _job = (LiveNodePacket) job;
            _resultHandler = resultHandler;
            _timeProvider = GetTimeProvider();
            _dataQueueHandler = GetDataQueueHandler();

            _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
            _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
            // sleep is controlled on this exchange via the GetNextTicksEnumerator
            _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
            _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
            _subscriptions = new ConcurrentDictionary<Symbol, List<Subscription>>();

            _bridge = new BusyBlockingCollection<TimeSlice>();
            _universeSelection = new UniverseSelection(this, algorithm, job.Controls);

            // run the exchanges
            Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
            Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));

            // this value will be modified via calls to AddSubscription/RemoveSubscription
            var ffres = Time.OneMinute;
            _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);

            // wire ourselves up to receive notifications when universes are added/removed
            var start = _timeProvider.GetUtcNow();
            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
            {
                switch (args.Action)
                {
                    case NotifyCollectionChangedAction.Add:
                        foreach (var universe in args.NewItems.OfType<Universe>())
                        {
                            _subscriptions.Add(universe.Configuration.Symbol, CreateUniverseSubscription(universe, start, Time.EndOfTime));
                            UpdateFillForwardResolution();
                        }
                        break;

                    case NotifyCollectionChangedAction.Remove:
                        foreach (var universe in args.OldItems.OfType<Universe>())
                        {
                            RemoveSubscription(universe.Configuration.Symbol);
                        }
                        break;

                    default:
                        throw new NotImplementedException("The specified action is not implemented: " + args.Action);
                }
            };
        }
 /// <summary>
 /// Initializes this history provider to work for the specified job
 /// </summary>
 /// <param name="job">The job</param>
 /// <param name="mapFileProvider">Provider used to get a map file resolver to handle equity mapping</param>
 /// <param name="factorFileProvider">Provider used to get factor files to handle equity price scaling</param>
 /// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
 /// <param name="statusUpdate">Function used to send status updates</param>
 public void Initialize(AlgorithmNodePacket job, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider, Action<int> statusUpdate)
 {
     return;
 }