Пример #1
0
        /// <summary>
        /// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
        /// </summary>
        public FakeDataQueue()
        {
            _aggregator = new AggregationManager();
            _symbols    = new HashSet <Symbol>();

            // load it up to start
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();

            _timer = new Timer
            {
                AutoReset = true,
                Enabled   = true,
                Interval  = 1000,
            };

            var lastCount = 0;
            var lastTime  = DateTime.Now;

            _timer.Elapsed += (sender, args) =>
            {
                var elapsed        = (DateTime.Now - lastTime);
                var ticksPerSecond = (count - lastCount) / elapsed.TotalSeconds;
                Console.WriteLine("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
                lastCount = count;
                lastTime  = DateTime.Now;
                PopulateQueue();
            };
        }
Пример #2
0
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="wssUrl">websockets url</param>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="passPhrase">pass phrase</param>
        /// <param name="algorithm">the algorithm instance is required to retreive account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">consolidate ticks</param>
        public GDAXBrokerage(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
                             IPriceProvider priceProvider, IDataAggregator aggregator)
            : base(wssUrl, websocket, restClient, apiKey, apiSecret, Market.GDAX, "GDAX")
        {
            FillSplit      = new ConcurrentDictionary <long, GDAXFill>();
            _passPhrase    = passPhrase;
            _algorithm     = algorithm;
            _priceProvider = priceProvider;
            _aggregator    = aggregator;

            WebSocket.Open += (sender, args) =>
            {
                var tickers = new[]
                {
                    "LTCUSD", "LTCEUR", "LTCBTC",
                    "BTCUSD", "BTCEUR", "BTCGBP",
                    "ETHBTC", "ETHUSD", "ETHEUR",
                    "BCHBTC", "BCHUSD", "BCHEUR",
                    "XRPUSD", "XRPEUR", "XRPBTC",
                    "EOSUSD", "EOSEUR", "EOSBTC",
                    "XLMUSD", "XLMEUR", "XLMBTC",
                    "ETCUSD", "ETCEUR", "ETCBTC",
                    "ZRXUSD", "ZRXEUR", "ZRXBTC",
                };
                Subscribe(tickers.Select(ticker => Symbol.Create(ticker, SecurityType.Crypto, Market.GDAX)));
            };

            _isDataQueueHandler = this is GDAXDataQueueHandler;
        }
Пример #3
0
        /// <summary>
        /// Initialize the instance of this class
        /// </summary>
        /// <param name="wssUrl">The web socket base url</param>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="passPhrase">pass phrase</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">the aggregator for consolidating ticks</param>
        /// <param name="job">The live job packet</param>
        protected void Initialize(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret,
                                  string passPhrase, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
        {
            if (IsInitialized)
            {
                return;
            }
            base.Initialize(wssUrl, websocket, restClient, apiKey, apiSecret);
            _job           = job;
            FillSplit      = new ConcurrentDictionary <long, GDAXFill>();
            _passPhrase    = passPhrase;
            _algorithm     = algorithm;
            _priceProvider = priceProvider;
            _aggregator    = aggregator;

            _isDataQueueHandler = this is GDAXDataQueueHandler;

            _fillMonitorTask = Task.Factory.StartNew(FillMonitorAction, _ctsFillMonitor.Token);

            var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();

            subscriptionManager.SubscribeImpl += (s, t) =>
            {
                Subscribe(s);
                return(true);
            };
            subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);

            SubscriptionManager = subscriptionManager;
        }
Пример #4
0
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">consolidate ticks</param>
        /// <param name="job">The live job packet</param>
        public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
            : base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
        {
            _job = job;

            SubscriptionManager = new BrokerageMultiWebSocketSubscriptionManager(
                WebSocketUrl,
                MaximumSymbolsPerConnection,
                0,
                null,
                () => new BitfinexWebSocketWrapper(null),
                Subscribe,
                Unsubscribe,
                OnDataMessage,
                TimeSpan.Zero,
                _connectionRateLimiter);

            _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
            _algorithm  = algorithm;
            _aggregator = aggregator;

            // load currency map
            using (var wc = new WebClient())
            {
                var json = wc.DownloadString("https://api-pub.bitfinex.com/v2/conf/pub:map:currency:sym");
                var rows = JsonConvert.DeserializeObject <List <List <List <string> > > >(json)[0];
                _currencyMap = rows
                               .ToDictionary(row => row[0], row => row[1].ToUpperInvariant());
            }

            WebSocket.Open += (sender, args) =>
            {
                SubscribeAuth();
            };
        }
Пример #5
0
        /// <summary>
        /// Initializes a new instance of the <see cref="OandaRestApiBase"/> class.
        /// </summary>
        /// <param name="symbolMapper">The symbol mapper.</param>
        /// <param name="orderProvider">The order provider.</param>
        /// <param name="securityProvider">The holdings provider.</param>
        /// <param name="aggregator">Consolidate ticks</param>
        /// <param name="environment">The Oanda environment (Trade or Practice)</param>
        /// <param name="accessToken">The Oanda access token (can be the user's personal access token or the access token obtained with OAuth by QC on behalf of the user)</param>
        /// <param name="accountId">The account identifier.</param>
        /// <param name="agent">The Oanda agent string</param>
        protected OandaRestApiBase(OandaSymbolMapper symbolMapper, IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, Environment environment, string accessToken, string accountId, string agent)
            : base("Oanda Brokerage")
        {
            SymbolMapper     = symbolMapper;
            OrderProvider    = orderProvider;
            SecurityProvider = securityProvider;
            Environment      = environment;
            AccessToken      = accessToken;
            AccountId        = accountId;
            Agent            = agent;
            Aggregator       = aggregator;

            PricingConnectionHandler = new DefaultConnectionHandler {
                MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
            };
            PricingConnectionHandler.ConnectionLost     += OnPricingConnectionLost;
            PricingConnectionHandler.ConnectionRestored += OnPricingConnectionRestored;
            PricingConnectionHandler.ReconnectRequested += OnPricingReconnectRequested;
            PricingConnectionHandler.Initialize(null);

            TransactionsConnectionHandler = new DefaultConnectionHandler {
                MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
            };
            TransactionsConnectionHandler.ConnectionLost     += OnTransactionsConnectionLost;
            TransactionsConnectionHandler.ConnectionRestored += OnTransactionsConnectionRestored;
            TransactionsConnectionHandler.ReconnectRequested += OnTransactionsReconnectRequested;
            TransactionsConnectionHandler.Initialize(null);
        }
Пример #6
0
 /// <summary>
 /// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
 /// </summary>
 public CoinApiDataQueueHandler(IDataAggregator dataAggregator)
 {
     _client             = new CoinApiWsClient();
     _client.TradeEvent += OnTrade;
     _client.QuoteEvent += OnQuote;
     _client.Error      += OnError;
     _dataAggregator     = dataAggregator;
 }
Пример #7
0
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="aggregator">the aggregator for consolidating ticks</param>
        /// <param name="job">The live job packet</param>
        public BinanceBrokerage(string apiKey, string apiSecret, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
            : base(WebSocketBaseUrl, new WebSocketClientWrapper(), null, apiKey, apiSecret, "Binance")
        {
            _job            = job;
            _algorithm      = algorithm;
            _aggregator     = aggregator;
            _messageHandler = new BrokerageConcurrentMessageHandler <WebSocketMessage>(OnMessageImpl);

            var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();

            subscriptionManager.SubscribeImpl += (s, t) =>
            {
                Subscribe(s);
                return(true);
            };
            subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);

            SubscriptionManager = subscriptionManager;

            _apiClient = new BinanceRestApiClient(
                _symbolMapper,
                algorithm?.Portfolio,
                apiKey,
                apiSecret);

            _apiClient.OrderSubmit        += (s, e) => OnOrderSubmit(e);
            _apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
            _apiClient.Message            += (s, e) => OnMessage(e);

            // User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
            _keepAliveTimer = new Timer
            {
                // 30 minutes
                Interval = 30 * 60 * 1000
            };
            _keepAliveTimer.Elapsed += (s, e) => _apiClient.SessionKeepAlive();

            WebSocket.Open   += (s, e) => { _keepAliveTimer.Start(); };
            WebSocket.Closed += (s, e) => { _keepAliveTimer.Stop(); };

            // A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
            _reconnectTimer = new Timer
            {
                // 23.5 hours
                Interval = 23.5 * 60 * 60 * 1000
            };
            _reconnectTimer.Elapsed += (s, e) =>
            {
                Log.Trace("Daily websocket restart: disconnect");
                Disconnect();

                Log.Trace("Daily websocket restart: connect");
                Connect();
            };
        }
 /// <summary>
 /// Constructs instance of <see cref="FrequencyCalculationService"/>.
 /// </summary>
 /// <param name="dataReader">Instance of <see cref="IDataReader"/> to read input data.</param>
 /// <param name="dataWriter">Instance of <see cref="IDataWriter"/> to write output data.</param>
 /// <param name="dataAggregator">Instance of <see cref="IDataAggregator"/> to aggregate results.</param>
 /// <param name="frequencyCalculator">Instance of <see cref="IFrequencyCalculator"/> to calculate frequencies.</param>
 public SingleThreadedFrequencyCalculationService(
     IDataReader dataReader,
     IDataWriter dataWriter,
     IDataAggregator dataAggregator,
     IFrequencyCalculator frequencyCalculator)
 {
     this.dataReader          = dataReader;
     this.dataWriter          = dataWriter;
     this.dataAggregator      = dataAggregator;
     this.frequencyCalculator = frequencyCalculator;
 }
Пример #9
0
        /// <summary>
        /// IQFeedDataQueueHandler is an implementation of IDataQueueHandler:
        /// </summary>
        public IQFeedDataQueueHandler(IDataAggregator aggregator)
        {
            _symbols     = new HashSet <Symbol>();
            _underlyings = new Dictionary <Symbol, Symbol>();
            _aggregator  = aggregator;

            if (!IsConnected)
            {
                Connect();
            }
        }
Пример #10
0
 public TestGDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey,
                                 string apiSecret,
                                 string passPhrase,
                                 IAlgorithm algorithm,
                                 IPriceProvider priceProvider,
                                 IDataAggregator aggregator,
                                 LiveNodePacket job
                                 )
     : base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, job)
 {
 }
Пример #11
0
        /// <summary>
        /// Initializes a new instance of the <see cref="OandaRestApiBase"/> class.
        /// </summary>
        /// <param name="symbolMapper">The symbol mapper.</param>
        /// <param name="orderProvider">The order provider.</param>
        /// <param name="securityProvider">The holdings provider.</param>
        /// <param name="aggregator">Consolidate ticks</param>
        /// <param name="environment">The Oanda environment (Trade or Practice)</param>
        /// <param name="accessToken">The Oanda access token (can be the user's personal access token or the access token obtained with OAuth by QC on behalf of the user)</param>
        /// <param name="accountId">The account identifier.</param>
        /// <param name="agent">The Oanda agent string</param>
        protected OandaRestApiBase(OandaSymbolMapper symbolMapper, IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, Environment environment, string accessToken, string accountId, string agent)
            : base("Oanda Brokerage")
        {
            SymbolMapper         = symbolMapper;
            OrderProvider        = orderProvider;
            SecurityProvider     = securityProvider;
            Environment          = environment;
            AccessToken          = accessToken;
            AccountId            = accountId;
            Agent                = agent;
            Aggregator           = aggregator;
            _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
            _subscriptionManager.SubscribeImpl   += (s, t) => Refresh();
            _subscriptionManager.UnsubscribeImpl += (s, t) => Refresh();

            PricingConnectionHandler = new DefaultConnectionHandler {
                MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
            };
            PricingConnectionHandler.ConnectionLost     += OnPricingConnectionLost;
            PricingConnectionHandler.ConnectionRestored += OnPricingConnectionRestored;
            PricingConnectionHandler.ReconnectRequested += OnPricingReconnectRequested;
            PricingConnectionHandler.Initialize(null);

            TransactionsConnectionHandler = new DefaultConnectionHandler {
                MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
            };
            TransactionsConnectionHandler.ConnectionLost     += OnTransactionsConnectionLost;
            TransactionsConnectionHandler.ConnectionRestored += OnTransactionsConnectionRestored;
            TransactionsConnectionHandler.ReconnectRequested += OnTransactionsReconnectRequested;
            TransactionsConnectionHandler.Initialize(null);

            Task.Factory.StartNew(
                () =>
            {
                do
                {
                    _refreshEvent.WaitOne();
                    Thread.Sleep(SubscribeDelay);

                    if (!_isConnected)
                    {
                        continue;
                    }

                    _refreshEvent.Reset();

                    var symbolsToSubscribe = SubscribedSymbols;
                    // restart streaming session
                    SubscribeSymbols(symbolsToSubscribe);
                } while (!_streamingCancellationTokenSource.IsCancellationRequested);
            },
                TaskCreationOptions.LongRunning
                );
        }
Пример #12
0
 public IngestionProcessor(IPersistentDataProvider persistence, IDataAggregator aggregator, IModelProvider model, IDataMessenger messenger, IRangeMerger <AggregatedDataRange> aggregatedMerger, IRangeMerger <TimeRange> timeMerger, IRangeFilterer <AggregatedDataRange> filterer, SeriesConfiguration configuration, ILogger <IngestionProcessor> logger)
 {
     _persistence                  = persistence;
     _aggregator                   = aggregator;
     _model                        = model;
     _messenger                    = messenger;
     _aggregatedMerger             = aggregatedMerger;
     _timeMerger                   = timeMerger;
     _filterer                     = filterer;
     _configuration                = configuration;
     _logger                       = logger;
     _telemetry.InstrumentationKey = "0def8f5e-9482-48ec-880d-4d2a81834a49";
 }
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">consolidate ticks</param>
        public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator)
            : base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
        {
            SubscriptionManager       = new BitfinexSubscriptionManager(this, WebSocketUrl, _symbolMapper);
            _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
            _algorithm  = algorithm;
            _aggregator = aggregator;

            WebSocket.Open += (sender, args) =>
            {
                SubscribeAuth();
            };
        }
Пример #14
0
        public CassandraPiSimDataProvider(CassandraConfiguration configuration, IDataAggregator aggregator, ILogger <CassandraPiSimDataProvider> logger, IRangeMerger <RawDataRange> rawMerger)
        {
            _configuration = configuration;
            _aggregator    = aggregator;
            _logger        = logger;
            _rawMerger     = rawMerger;
            var cluster = Cluster.Builder()
                          .AddContactPoints(configuration.Address)
                          .WithDefaultKeyspace(configuration.Keyspace)
                          .WithQueryTimeout(configuration.QueryTimeout)
                          .Build();

            _session = cluster.Connect();
        }
Пример #15
0
        /// <summary>
        /// Initializes a new instance of the <see cref="AlpacaBrokerage"/> class.
        /// </summary>
        /// <param name="orderProvider">The order provider.</param>
        /// <param name="securityProvider">The holdings provider.</param>
        /// <param name="accountKeyId">The Alpaca api key id</param>
        /// <param name="secretKey">The api secret key</param>
        /// <param name="tradingMode">The Alpaca trading mode. paper/live</param>
        /// <param name="handlesMarketData">true if market data subscriptions will be handled by Alpaca</param>
        /// <param name="aggregator">consolidate ticks</param>
        public AlpacaBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, string accountKeyId, string secretKey, string tradingMode, bool handlesMarketData, IDataAggregator aggregator)
            : base("Alpaca Brokerage")
        {
            _handlesMarketData = handlesMarketData;
            _aggregator        = aggregator;

            var httpScheme    = "https://";
            var alpacaBaseUrl = "api.alpaca.markets";

            if (tradingMode.Equals("paper"))
            {
                alpacaBaseUrl = "paper-" + alpacaBaseUrl;
            }

            var httpAlpacaBaseUrl = httpScheme + alpacaBaseUrl;

            _orderProvider    = orderProvider;
            _securityProvider = securityProvider;

            _marketHours = MarketHoursDatabase.FromDataFolder();

            // Alpaca trading client
            _alpacaTradingClient = new AlpacaTradingClient(new AlpacaTradingClientConfiguration
            {
                ApiEndpoint = tradingMode.Equals("paper") ? Environments.Paper.AlpacaTradingApi : Environments.Live.AlpacaTradingApi,
                SecurityId  = new SecretKey(accountKeyId, secretKey)
            });
            // api client for alpaca data
            _polygonDataClient = new PolygonDataClient(new PolygonDataClientConfiguration
            {
                ApiEndpoint = Environments.Live.PolygonDataApi,
                KeyId       = accountKeyId
            });

            // websocket client for alpaca
            _sockClient = new SockClient(accountKeyId, secretKey, httpAlpacaBaseUrl);
            _sockClient.OnTradeUpdate += OnTradeUpdate;
            _sockClient.OnError       += OnSockClientError;

            // Polygon Streaming client for Alpaca (streams trade and quote data)
            _polygonStreamingClient = new PolygonStreamingClient(new PolygonStreamingClientConfiguration
            {
                ApiEndpoint      = Environments.Live.PolygonStreamingApi,
                KeyId            = accountKeyId,
                WebSocketFactory = new WebSocketClientFactory()
            });
            _polygonStreamingClient.QuoteReceived += OnQuoteReceived;
            _polygonStreamingClient.TradeReceived += OnTradeReceived;
            _polygonStreamingClient.OnError       += OnPolygonStreamingClientError;
        }
Пример #16
0
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="wssUrl">websockets url</param>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="passPhrase">pass phrase</param>
        /// <param name="algorithm">the algorithm instance is required to retreive account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">consolidate ticks</param>
        public GDAXBrokerage(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
                             IPriceProvider priceProvider, IDataAggregator aggregator)
            : base(wssUrl, websocket, restClient, apiKey, apiSecret, "GDAX")
        {
            FillSplit      = new ConcurrentDictionary <long, GDAXFill>();
            _passPhrase    = passPhrase;
            _algorithm     = algorithm;
            _priceProvider = priceProvider;
            _aggregator    = aggregator;

            _isDataQueueHandler = this is GDAXDataQueueHandler;

            _fillMonitorTask = Task.Factory.StartNew(FillMonitorAction, _ctsFillMonitor.Token);
        }
Пример #17
0
        /// <summary>
        /// Initializes a new instance of the <see cref="GDAXDataQueueHandler"/> class
        /// </summary>
        public GDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
                                    IPriceProvider priceProvider, IDataAggregator aggregator)
            : base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator)
        {
            var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();

            subscriptionManager.SubscribeImpl += (s, t) =>
            {
                Subscribe(s);
                return(true);
            };
            subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);

            SubscriptionManager = subscriptionManager;
        }
Пример #18
0
        public IEXDataQueueHandler(IDataAggregator aggregator, bool live)
        {
            _aggregator = aggregator;
            Endpoint    = "https://ws-api.iextrading.com/1.0/tops";

            if (string.IsNullOrWhiteSpace(_apiKey))
            {
                Log.Trace("IEXDataQueueHandler(): The IEX API key was not provided, history calls will return no data.");
            }

            if (live)
            {
                Reconnect();
            }
        }
Пример #19
0
        /// <summary>
        /// Creates a new instance of the <see cref="FxcmBrokerage"/> class
        /// </summary>
        /// <param name="orderProvider">The order provider</param>
        /// <param name="securityProvider">The holdings provider</param>
        /// <param name="aggregator">Consolidate ticks</param>
        /// <param name="server">The url of the server</param>
        /// <param name="terminal">The terminal name</param>
        /// <param name="userName">The user name (login id)</param>
        /// <param name="password">The user password</param>
        /// <param name="accountId">The account id</param>
        public FxcmBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, string server, string terminal, string userName, string password, string accountId)
            : base("FXCM Brokerage")
        {
            _orderProvider    = orderProvider;
            _securityProvider = securityProvider;
            _aggregator       = aggregator;
            _server           = server;
            _terminal         = terminal;
            _userName         = userName;
            _password         = password;
            _accountId        = accountId;

            HistoryResponseTimeout      = 5000;
            MaximumHistoryRetryAttempts = 1;
        }
Пример #20
0
        /// <summary>
        /// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
        /// </summary>
        public FakeDataQueue(IDataAggregator dataAggregator)
        {
            _aggregator        = dataAggregator;
            _dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), true);
            var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"), false);

            _optionChainProvider                  = new LiveOptionChainProvider(_dataCacheProvider, mapFileProvider);
            _marketHoursDatabase                  = MarketHoursDatabase.FromDataFolder();
            _symbolExchangeTimeZones              = new Dictionary <Symbol, TimeZoneOffsetProvider>();
            _subscriptionManager                  = new EventBasedDataQueueHandlerSubscriptionManager();
            _subscriptionManager.SubscribeImpl   += (s, t) => true;
            _subscriptionManager.UnsubscribeImpl += (s, t) => true;

            // load it up to start
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();

            _timer = new Timer
            {
                AutoReset = false,
                Enabled   = true,
                Interval  = 1000,
            };

            var lastCount = 0;
            var lastTime  = DateTime.UtcNow;

            _timer.Elapsed += (sender, args) =>
            {
                var elapsed        = (DateTime.UtcNow - lastTime);
                var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
                Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
                lastCount = _count;
                lastTime  = DateTime.UtcNow;
                PopulateQueue();
                try
                {
                    _timer.Reset();
                }
                catch (ObjectDisposedException)
                {
                    // pass
                }
            };
        }
Пример #21
0
 /// <summary>
 /// Initializes a new instance of the <see cref="GDAXDataQueueHandler"/> class
 /// </summary>
 public GDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
                             IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
     : base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, job)
 {
     Initialize(
         wssUrl: wssUrl,
         websocket: websocket,
         restClient: restClient,
         apiKey: apiKey,
         apiSecret: apiSecret,
         passPhrase: passPhrase,
         algorithm: algorithm,
         priceProvider: priceProvider,
         aggregator: aggregator,
         job: job
         );
 }
        public AnomalyDetection(IOrchestrator orcherstratorProxy, IDataAggregator aggregatorProxy)
        {
            orcherstratorProxy.Detection.Subscribe(this, async signal =>
            {
                int cluster = 0;
                if (_kMeansClustering != null)
                {
                    lock (_syncClustering)
                        if (_kMeansClustering != null)
                        {
                            cluster = _kMeansClustering.Classify(new double[] { signal.Value });
                        }
                }

                if (cluster < 0)
                {
                    System.Console.WriteLine("_____________________________Anomaly detected__________________________________");
                    await Anomaly.PublishAsync(new Anomaly()
                    {
                        Temperature = signal
                    });
                }
                return(MessageResult.Ok);
            });

            aggregatorProxy.Aggregate.Subscribe(this, async(sampleReference) =>
            {
                //if the messages has been stored and forwarded, but the file has been deleted (e.g. a restart)
                //then the message can be empty (null)
                if (sampleReference == null)
                {
                    return(MessageResult.Ok);
                }

                System.Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");

                lock (_syncSample)
                    _sample = sampleReference.Message.Values;

                lock (_syncClustering)
                    _kMeansClustering = new KMeansClustering(_sample, _numClusters);

                return(MessageResult.Ok);
            });
        }
Пример #23
0
        /// <summary>
        /// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
        /// </summary>
        public FakeDataQueue(IDataAggregator dataAggregator)
        {
            _aggregator                           = dataAggregator;
            _marketHoursDatabase                  = MarketHoursDatabase.FromDataFolder();
            _symbolExchangeTimeZones              = new Dictionary <Symbol, TimeZoneOffsetProvider>();
            _subscriptionManager                  = new EventBasedDataQueueHandlerSubscriptionManager();
            _subscriptionManager.SubscribeImpl   += (s, t) => true;
            _subscriptionManager.UnsubscribeImpl += (s, t) => true;

            // load it up to start
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();
            PopulateQueue();

            _timer = new Timer
            {
                AutoReset = false,
                Enabled   = true,
                Interval  = 1000,
            };

            var lastCount = 0;
            var lastTime  = DateTime.UtcNow;

            _timer.Elapsed += (sender, args) =>
            {
                var elapsed        = (DateTime.UtcNow - lastTime);
                var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
                Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
                lastCount = _count;
                lastTime  = DateTime.UtcNow;
                PopulateQueue();
                try
                {
                    _timer.Reset();
                }
                catch (ObjectDisposedException)
                {
                    // pass
                }
            };
        }
Пример #24
0
        /// <summary>
        /// Creates a new instance of the <see cref="FxcmBrokerage"/> class
        /// </summary>
        /// <param name="orderProvider">The order provider</param>
        /// <param name="securityProvider">The holdings provider</param>
        /// <param name="aggregator">Consolidate ticks</param>
        /// <param name="server">The url of the server</param>
        /// <param name="terminal">The terminal name</param>
        /// <param name="userName">The user name (login id)</param>
        /// <param name="password">The user password</param>
        /// <param name="accountId">The account id</param>
        public FxcmBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, string server, string terminal, string userName, string password, string accountId)
            : base("FXCM Brokerage")
        {
            _orderProvider    = orderProvider;
            _securityProvider = securityProvider;
            _aggregator       = aggregator;
            _server           = server;
            _terminal         = terminal;
            _userName         = userName;
            _password         = password;
            _accountId        = accountId;

            _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
            _subscriptionManager.SubscribeImpl   += (s, t) => Subscribe(s);
            _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);

            HistoryResponseTimeout      = 5000;
            MaximumHistoryRetryAttempts = 1;
        }
Пример #25
0
        /// <summary>
        /// Constructor for brokerage
        /// </summary>
        /// <param name="websocket">instance of websockets client</param>
        /// <param name="restClient">instance of rest client</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
        /// <param name="aggregator">consolidate ticks</param>
        public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator)
            : base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
        {
            SubscriptionManager       = new BitfinexSubscriptionManager(this, WebSocketUrl, _symbolMapper);
            _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
            _algorithm  = algorithm;
            _aggregator = aggregator;

            // load currency map
            using (var wc = new WebClient())
            {
                var json = wc.DownloadString("https://api-pub.bitfinex.com/v2/conf/pub:map:currency:sym");
                var rows = JsonConvert.DeserializeObject <List <List <List <string> > > >(json)[0];
                _currencyMap = rows
                               .ToDictionary(row => row[0], row => row[1].ToUpperInvariant());
            }

            WebSocket.Open += (sender, args) =>
            {
                SubscribeAuth();
            };
        }
Пример #26
0
        public Level1Port(IDataAggregator aggregator, IQFeedDataQueueUniverseProvider symbolUniverse)
            : base(80)
        {
            start          = DateTime.Now;
            _prices        = new ConcurrentDictionary <string, double>();
            _openInterests = new ConcurrentDictionary <string, int>();

            _aggregator                    = aggregator;
            _symbolUniverse                = symbolUniverse;
            Level1SummaryUpdateEvent      += OnLevel1SummaryUpdateEvent;
            Level1TimerEvent              += OnLevel1TimerEvent;
            Level1ServerDisconnectedEvent += OnLevel1ServerDisconnected;
            Level1ServerReconnectFailed   += OnLevel1ServerReconnectFailed;
            Level1UnknownEvent            += OnLevel1UnknownEvent;
            Level1FundamentalEvent        += OnLevel1FundamentalEvent;

            _timer           = new Timer(1000);
            _timer.Enabled   = false;
            _timer.AutoReset = true;
            _timer.Elapsed  += (sender, args) =>
            {
                var ticksPerSecond = count / (DateTime.Now - start).TotalSeconds;
                int dataQueueCount = Interlocked.Exchange(ref _dataQueueCount, 0);
                if (ticksPerSecond > 1000 || dataQueueCount > 31)
                {
                    Log.Trace($"IQFeed.OnSecond(): Ticks/sec: {ticksPerSecond.ToStringInvariant("0000.00")} " +
                              $"Engine.Ticks.Count: {dataQueueCount} CPU%: {OS.CpuUsage.ToStringInvariant("0.0") + "%"}"
                              );
                }

                count = 0;
                start = DateTime.Now;
            };

            _timer.Enabled = true;
        }
Пример #27
0
        /// <summary>
        /// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
        /// </summary>
        public CoinApiDataQueueHandler()
        {
            _dataAggregator = Composer.Instance.GetPart <IDataAggregator>();
            if (_dataAggregator == null)
            {
                _dataAggregator =
                    Composer.Instance.GetExportedValueByTypeName <IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
            }
            var product = Config.GetValue <CoinApiProduct>("coinapi-product");

            _streamingDataType = product < CoinApiProduct.Streamer
                ? new[] { "trade" }
                : new[] { "trade", "quote" };

            Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'");

            _client              = new CoinApiWsClient();
            _client.TradeEvent  += OnTrade;
            _client.QuoteEvent  += OnQuote;
            _client.Error       += OnError;
            _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
            _subscriptionManager.SubscribeImpl   += (s, t) => Subscribe(s);
            _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
        }
Пример #28
0
 public GDAXFakeDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
     IPriceProvider priceProvider, IDataAggregator aggregator)
 : base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, null)
 {
 }
Пример #29
0
 /// <summary>
 /// Constructor for brokerage
 /// </summary>
 /// <param name="apiKey">api key</param>
 /// <param name="apiSecret">api secret</param>
 /// <param name="restApiUrl">The rest api url</param>
 /// <param name="webSocketBaseUrl">The web socket base url</param>
 /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
 /// <param name="aggregator">the aggregator for consolidating ticks</param>
 /// <param name="job">The live job packet</param>
 public BinanceBrokerage(string apiKey, string apiSecret, string restApiUrl, string webSocketBaseUrl, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
     : base("Binance")
 {
     Initialize(
         wssUrl: webSocketBaseUrl,
         restApiUrl: restApiUrl,
         apiKey: apiKey,
         apiSecret: apiSecret,
         algorithm: algorithm,
         aggregator: aggregator,
         job: job
         );
 }
Пример #30
0
        /// <summary>
        /// Initialize the instance of this class
        /// </summary>
        /// <param name="wssUrl">The web socket base url</param>
        /// <param name="restApiUrl">The rest api url</param>
        /// <param name="apiKey">api key</param>
        /// <param name="apiSecret">api secret</param>
        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
        /// <param name="aggregator">the aggregator for consolidating ticks</param>
        /// <param name="job">The live job packet</param>
        private void Initialize(string wssUrl, string restApiUrl, string apiKey, string apiSecret,
                                IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
        {
            if (IsInitialized)
            {
                return;
            }
            base.Initialize(wssUrl, new WebSocketClientWrapper(), null, apiKey, apiSecret);
            _job              = job;
            _algorithm        = algorithm;
            _aggregator       = aggregator;
            _webSocketBaseUrl = wssUrl;
            _messageHandler   = new BrokerageConcurrentMessageHandler <WebSocketMessage>(OnUserMessage);

            var maximumWebSocketConnections = Config.GetInt("binance-maximum-websocket-connections");
            var symbolWeights = maximumWebSocketConnections > 0 ? FetchSymbolWeights() : null;

            var subscriptionManager = new BrokerageMultiWebSocketSubscriptionManager(
                wssUrl,
                MaximumSymbolsPerConnection,
                maximumWebSocketConnections,
                symbolWeights,
                () => new BinanceWebSocketWrapper(null),
                Subscribe,
                Unsubscribe,
                OnDataMessage,
                new TimeSpan(23, 45, 0));

            SubscriptionManager = subscriptionManager;

            // can be null, if BinanceBrokerage is used as DataQueueHandler only
            if (_algorithm != null)
            {
                // Binance rest api endpoint is different for sport and margin trading
                // we need to delay initialization of rest api client until Algorithm is initialized
                // and user brokerage choise is actually applied
                _apiClientLazy = new Lazy <BinanceBaseRestApiClient>(() =>
                {
                    BinanceBaseRestApiClient apiClient = _algorithm.BrokerageModel.AccountType == AccountType.Cash
                        ? new BinanceSpotRestApiClient(_symbolMapper, algorithm?.Portfolio, apiKey, apiSecret, restApiUrl)
                        : new BinanceCrossMarginRestApiClient(_symbolMapper, algorithm?.Portfolio, apiKey, apiSecret,
                                                              restApiUrl);

                    apiClient.OrderSubmit        += (s, e) => OnOrderSubmit(e);
                    apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
                    apiClient.Message            += (s, e) => OnMessage(e);

                    // once we know the api endpoint we can subscribe to user data stream
                    apiClient.CreateListenKey();
                    _keepAliveTimer.Elapsed += (s, e) => apiClient.SessionKeepAlive();

                    Connect(apiClient.SessionId);

                    return(apiClient);
                });
            }

            // User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
            _keepAliveTimer = new Timer
            {
                // 30 minutes
                Interval = 30 * 60 * 1000
            };

            WebSocket.Open += (s, e) =>
            {
                _keepAliveTimer.Start();
            };
            WebSocket.Closed += (s, e) =>
            {
                ApiClient.StopSession();
                _keepAliveTimer.Stop();
            };

            // A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
            _reconnectTimer = new Timer
            {
                // 23.5 hours
                Interval = 23.5 * 60 * 60 * 1000
            };
            _reconnectTimer.Elapsed += (s, e) =>
            {
                Log.Trace("Daily websocket restart: disconnect");
                Disconnect();

                Log.Trace("Daily websocket restart: connect");
                Connect();
            };
        }