public IEXDataQueueHandler(bool live)
{
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
_subscriptionManager.UnsubscribeImpl += (s, t) =>
{
Unsubscribe(s);
return(true);
};
Endpoint = "https://ws-api.iextrading.com/1.0/tops";
if (string.IsNullOrWhiteSpace(_apiKey))
{
Log.Trace("IEXDataQueueHandler(): The IEX API key was not provided, history calls will return no data.");
}
if (live)
{
Reconnect();
}
}
/// <summary>
/// Initialize the instance of this class
/// </summary>
/// <param name="wssUrl">The web socket base url</param>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="passPhrase">pass phrase</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
protected void Initialize(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret,
string passPhrase, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
{
if (IsInitialized)
{
return;
}
base.Initialize(wssUrl, websocket, restClient, apiKey, apiSecret);
_job = job;
FillSplit = new ConcurrentDictionary <long, GDAXFill>();
_passPhrase = passPhrase;
_algorithm = algorithm;
_priceProvider = priceProvider;
_aggregator = aggregator;
_isDataQueueHandler = this is GDAXDataQueueHandler;
_fillMonitorTask = Task.Factory.StartNew(FillMonitorAction, _ctsFillMonitor.Token);
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
public BinanceBrokerage(string apiKey, string apiSecret, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
: base(WebSocketBaseUrl, new WebSocketClientWrapper(), null, apiKey, apiSecret, "Binance")
{
_job = job;
_algorithm = algorithm;
_aggregator = aggregator;
_messageHandler = new BrokerageConcurrentMessageHandler <WebSocketMessage>(OnMessageImpl);
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
_apiClient = new BinanceRestApiClient(
_symbolMapper,
algorithm?.Portfolio,
apiKey,
apiSecret);
_apiClient.OrderSubmit += (s, e) => OnOrderSubmit(e);
_apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
_apiClient.Message += (s, e) => OnMessage(e);
// User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
_keepAliveTimer = new Timer
{
// 30 minutes
Interval = 30 * 60 * 1000
};
_keepAliveTimer.Elapsed += (s, e) => _apiClient.SessionKeepAlive();
WebSocket.Open += (s, e) => { _keepAliveTimer.Start(); };
WebSocket.Closed += (s, e) => { _keepAliveTimer.Stop(); };
// A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
_reconnectTimer = new Timer
{
// 23.5 hours
Interval = 23.5 * 60 * 60 * 1000
};
_reconnectTimer.Elapsed += (s, e) =>
{
Log.Trace("Daily websocket restart: disconnect");
Disconnect();
Log.Trace("Daily websocket restart: connect");
Connect();
};
}
/// <summary>
/// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
/// </summary>
public CoinApiDataQueueHandler()
{
_client = new CoinApiWsClient();
_client.TradeEvent += OnTrade;
_client.QuoteEvent += OnQuote;
_client.Error += OnError;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
}
/// <summary>
/// Initializes a new instance of the <see cref="OandaRestApiBase"/> class.
/// </summary>
/// <param name="symbolMapper">The symbol mapper.</param>
/// <param name="orderProvider">The order provider.</param>
/// <param name="securityProvider">The holdings provider.</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="environment">The Oanda environment (Trade or Practice)</param>
/// <param name="accessToken">The Oanda access token (can be the user's personal access token or the access token obtained with OAuth by QC on behalf of the user)</param>
/// <param name="accountId">The account identifier.</param>
/// <param name="agent">The Oanda agent string</param>
protected OandaRestApiBase(OandaSymbolMapper symbolMapper, IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, Environment environment, string accessToken, string accountId, string agent)
: base("Oanda Brokerage")
{
SymbolMapper = symbolMapper;
OrderProvider = orderProvider;
SecurityProvider = securityProvider;
Environment = environment;
AccessToken = accessToken;
AccountId = accountId;
Agent = agent;
Aggregator = aggregator;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Refresh();
_subscriptionManager.UnsubscribeImpl += (s, t) => Refresh();
PricingConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
PricingConnectionHandler.ConnectionLost += OnPricingConnectionLost;
PricingConnectionHandler.ConnectionRestored += OnPricingConnectionRestored;
PricingConnectionHandler.ReconnectRequested += OnPricingReconnectRequested;
PricingConnectionHandler.Initialize(null);
TransactionsConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
TransactionsConnectionHandler.ConnectionLost += OnTransactionsConnectionLost;
TransactionsConnectionHandler.ConnectionRestored += OnTransactionsConnectionRestored;
TransactionsConnectionHandler.ReconnectRequested += OnTransactionsReconnectRequested;
TransactionsConnectionHandler.Initialize(null);
Task.Factory.StartNew(
() =>
{
do
{
_refreshEvent.WaitOne();
Thread.Sleep(SubscribeDelay);
if (!_isConnected)
{
continue;
}
_refreshEvent.Reset();
var symbolsToSubscribe = SubscribedSymbols;
// restart streaming session
SubscribeSymbols(symbolsToSubscribe);
} while (!_streamingCancellationTokenSource.IsCancellationRequested);
},
TaskCreationOptions.LongRunning
);
}
/// <summary>
/// Initializes a new instance of the <see cref="GDAXDataQueueHandler"/> class
/// </summary>
public GDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator)
{
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
}
/// <summary>
/// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
/// </summary>
public FakeDataQueue(IDataAggregator dataAggregator)
{
_aggregator = dataAggregator;
_dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), true);
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"), false);
_optionChainProvider = new LiveOptionChainProvider(_dataCacheProvider, mapFileProvider);
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
_symbolExchangeTimeZones = new Dictionary <Symbol, TimeZoneOffsetProvider>();
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => true;
_subscriptionManager.UnsubscribeImpl += (s, t) => true;
// load it up to start
PopulateQueue();
PopulateQueue();
PopulateQueue();
PopulateQueue();
_timer = new Timer
{
AutoReset = false,
Enabled = true,
Interval = 1000,
};
var lastCount = 0;
var lastTime = DateTime.UtcNow;
_timer.Elapsed += (sender, args) =>
{
var elapsed = (DateTime.UtcNow - lastTime);
var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
lastCount = _count;
lastTime = DateTime.UtcNow;
PopulateQueue();
try
{
_timer.Reset();
}
catch (ObjectDisposedException)
{
// pass
}
};
}
/// <summary>
/// Initializes a new instance of the <see cref="PolygonDataQueueHandler"/> class
/// </summary>
public PolygonDataQueueHandler(bool streamingEnabled)
{
if (streamingEnabled)
{
foreach (var securityType in new[] { SecurityType.Equity, SecurityType.Forex, SecurityType.Crypto })
{
var client = new PolygonWebSocketClientWrapper(_apiKey, _symbolMapper, securityType, OnMessage);
_webSocketClientWrappers.Add(securityType, client);
}
}
var subscriber = new EventBasedDataQueueHandlerSubscriptionManager(t => t.ToString());
subscriber.SubscribeImpl += Subscribe;
subscriber.UnsubscribeImpl += Unsubscribe;
_subscriptionManager = subscriber;
}
/// <summary>
/// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
/// </summary>
public CoinApiDataQueueHandler()
{
var product = Config.GetValue <CoinApiProduct>("coinapi-product");
_streamingDataType = product < CoinApiProduct.Streamer
? new[] { "trade" }
: new[] { "trade", "quote" };
Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'");
_client = new CoinApiWsClient();
_client.TradeEvent += OnTrade;
_client.QuoteEvent += OnQuote;
_client.Error += OnError;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
}
/// <summary>
/// Creates a new instance of the <see cref="FxcmBrokerage"/> class
/// </summary>
/// <param name="orderProvider">The order provider</param>
/// <param name="securityProvider">The holdings provider</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="server">The url of the server</param>
/// <param name="terminal">The terminal name</param>
/// <param name="userName">The user name (login id)</param>
/// <param name="password">The user password</param>
/// <param name="accountId">The account id</param>
public FxcmBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, string server, string terminal, string userName, string password, string accountId)
: base("FXCM Brokerage")
{
_orderProvider = orderProvider;
_securityProvider = securityProvider;
_aggregator = aggregator;
_server = server;
_terminal = terminal;
_userName = userName;
_password = password;
_accountId = accountId;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
HistoryResponseTimeout = 5000;
MaximumHistoryRetryAttempts = 1;
}
/// <summary>
/// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
/// </summary>
public FakeDataQueue(IDataAggregator dataAggregator)
{
_aggregator = dataAggregator;
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
_symbolExchangeTimeZones = new Dictionary <Symbol, TimeZoneOffsetProvider>();
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => true;
_subscriptionManager.UnsubscribeImpl += (s, t) => true;
// load it up to start
PopulateQueue();
PopulateQueue();
PopulateQueue();
PopulateQueue();
_timer = new Timer
{
AutoReset = false,
Enabled = true,
Interval = 1000,
};
var lastCount = 0;
var lastTime = DateTime.UtcNow;
_timer.Elapsed += (sender, args) =>
{
var elapsed = (DateTime.UtcNow - lastTime);
var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
lastCount = _count;
lastTime = DateTime.UtcNow;
PopulateQueue();
try
{
_timer.Reset();
}
catch (ObjectDisposedException)
{
// pass
}
};
}
/// <summary>
/// IQFeedDataQueueHandler is an implementation of IDataQueueHandler:
/// </summary>
public IQFeedDataQueueHandler()
{
_symbols = new HashSet <Symbol>();
_underlyings = new Dictionary <Symbol, Symbol>();
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
_subscriptionManager.UnsubscribeImpl += (s, t) =>
{
Unsubscribe(s);
return(true);
};
if (!IsConnected)
{
Connect();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
/// </summary>
public CoinApiDataQueueHandler()
{
_dataAggregator = Composer.Instance.GetPart <IDataAggregator>();
if (_dataAggregator == null)
{
_dataAggregator =
Composer.Instance.GetExportedValueByTypeName <IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
}
var product = Config.GetValue <CoinApiProduct>("coinapi-product");
_streamingDataType = product < CoinApiProduct.Streamer
? new[] { "trade" }
: new[] { "trade", "quote" };
Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'");
_client = new CoinApiWsClient();
_client.TradeEvent += OnTrade;
_client.QuoteEvent += OnQuote;
_client.Error += OnError;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
}
/// <summary>
/// Initializes a new instance of the <see cref="IEXDataQueueHandler"/> class.
/// </summary>
public IEXDataQueueHandler()
{
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (symbols, t) =>
{
symbols.DoForEach(symbol =>
{
if (!_symbols.TryAdd(symbol.Value, symbol))
{
throw new InvalidOperationException($"Invalid logic, SubscriptionManager tried to subscribe to existing symbol : {symbol.Value}");
}
});
Refresh();
return(true);
};
_subscriptionManager.UnsubscribeImpl += (symbols, t) =>
{
symbols.DoForEach(symbol =>
{
Symbol tmp;
_symbols.TryRemove(symbol.Value, out tmp);
});
Refresh();
return(true);
};
if (string.IsNullOrWhiteSpace(_apiKey))
{
throw new ArgumentException("Could not read IEX API key from config.json. " +
"Please make sure to add \"iex-cloud-api-key\" to your configuration file");
}
// Set the sse-clients collection
_clients = new IEXEventSourceCollection(((o, args) =>
{
var message = args.Message.Data;
ProcessJsonObject(message);
}), _apiKey);
// In this thread, we check at each interval whether the client needs to be updated
// Subscription renewal requests may come in dozens and all at relatively same time - we cannot update them one by one when work with SSE
var clientUpdateThread = new Thread(() =>
{
while (!_cts.Token.IsCancellationRequested)
{
_refreshEvent.WaitOne();
Thread.Sleep(SubscribeDelay);
_refreshEvent.Reset();
try
{
_clients.UpdateSubscription(_symbols.Keys.ToArray());
}
catch (Exception e)
{
Log.Error(e);
throw;
}
}
})
{
IsBackground = true
};
clientUpdateThread.Start();
}
