/// <summary>
/// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
/// </summary>
public FakeDataQueue()
{
_aggregator = new AggregationManager();
_symbols = new HashSet <Symbol>();
// load it up to start
PopulateQueue();
PopulateQueue();
PopulateQueue();
PopulateQueue();
_timer = new Timer
{
AutoReset = true,
Enabled = true,
Interval = 1000,
};
var lastCount = 0;
var lastTime = DateTime.Now;
_timer.Elapsed += (sender, args) =>
{
var elapsed = (DateTime.Now - lastTime);
var ticksPerSecond = (count - lastCount) / elapsed.TotalSeconds;
Console.WriteLine("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
lastCount = count;
lastTime = DateTime.Now;
PopulateQueue();
};
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="wssUrl">websockets url</param>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="passPhrase">pass phrase</param>
/// <param name="algorithm">the algorithm instance is required to retreive account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">consolidate ticks</param>
public GDAXBrokerage(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, Market.GDAX, "GDAX")
{
FillSplit = new ConcurrentDictionary <long, GDAXFill>();
_passPhrase = passPhrase;
_algorithm = algorithm;
_priceProvider = priceProvider;
_aggregator = aggregator;
WebSocket.Open += (sender, args) =>
{
var tickers = new[]
{
"LTCUSD", "LTCEUR", "LTCBTC",
"BTCUSD", "BTCEUR", "BTCGBP",
"ETHBTC", "ETHUSD", "ETHEUR",
"BCHBTC", "BCHUSD", "BCHEUR",
"XRPUSD", "XRPEUR", "XRPBTC",
"EOSUSD", "EOSEUR", "EOSBTC",
"XLMUSD", "XLMEUR", "XLMBTC",
"ETCUSD", "ETCEUR", "ETCBTC",
"ZRXUSD", "ZRXEUR", "ZRXBTC",
};
Subscribe(tickers.Select(ticker => Symbol.Create(ticker, SecurityType.Crypto, Market.GDAX)));
};
_isDataQueueHandler = this is GDAXDataQueueHandler;
}
/// <summary>
/// Initialize the instance of this class
/// </summary>
/// <param name="wssUrl">The web socket base url</param>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="passPhrase">pass phrase</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
protected void Initialize(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret,
string passPhrase, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
{
if (IsInitialized)
{
return;
}
base.Initialize(wssUrl, websocket, restClient, apiKey, apiSecret);
_job = job;
FillSplit = new ConcurrentDictionary <long, GDAXFill>();
_passPhrase = passPhrase;
_algorithm = algorithm;
_priceProvider = priceProvider;
_aggregator = aggregator;
_isDataQueueHandler = this is GDAXDataQueueHandler;
_fillMonitorTask = Task.Factory.StartNew(FillMonitorAction, _ctsFillMonitor.Token);
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">consolidate ticks</param>
/// <param name="job">The live job packet</param>
public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
: base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
{
_job = job;
SubscriptionManager = new BrokerageMultiWebSocketSubscriptionManager(
WebSocketUrl,
MaximumSymbolsPerConnection,
0,
null,
() => new BitfinexWebSocketWrapper(null),
Subscribe,
Unsubscribe,
OnDataMessage,
TimeSpan.Zero,
_connectionRateLimiter);
_symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
_algorithm = algorithm;
_aggregator = aggregator;
// load currency map
using (var wc = new WebClient())
{
var json = wc.DownloadString("https://api-pub.bitfinex.com/v2/conf/pub:map:currency:sym");
var rows = JsonConvert.DeserializeObject <List <List <List <string> > > >(json)[0];
_currencyMap = rows
.ToDictionary(row => row[0], row => row[1].ToUpperInvariant());
}
WebSocket.Open += (sender, args) =>
{
SubscribeAuth();
};
}
/// <summary>
/// Initializes a new instance of the <see cref="OandaRestApiBase"/> class.
/// </summary>
/// <param name="symbolMapper">The symbol mapper.</param>
/// <param name="orderProvider">The order provider.</param>
/// <param name="securityProvider">The holdings provider.</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="environment">The Oanda environment (Trade or Practice)</param>
/// <param name="accessToken">The Oanda access token (can be the user's personal access token or the access token obtained with OAuth by QC on behalf of the user)</param>
/// <param name="accountId">The account identifier.</param>
/// <param name="agent">The Oanda agent string</param>
protected OandaRestApiBase(OandaSymbolMapper symbolMapper, IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, Environment environment, string accessToken, string accountId, string agent)
: base("Oanda Brokerage")
{
SymbolMapper = symbolMapper;
OrderProvider = orderProvider;
SecurityProvider = securityProvider;
Environment = environment;
AccessToken = accessToken;
AccountId = accountId;
Agent = agent;
Aggregator = aggregator;
PricingConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
PricingConnectionHandler.ConnectionLost += OnPricingConnectionLost;
PricingConnectionHandler.ConnectionRestored += OnPricingConnectionRestored;
PricingConnectionHandler.ReconnectRequested += OnPricingReconnectRequested;
PricingConnectionHandler.Initialize(null);
TransactionsConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
TransactionsConnectionHandler.ConnectionLost += OnTransactionsConnectionLost;
TransactionsConnectionHandler.ConnectionRestored += OnTransactionsConnectionRestored;
TransactionsConnectionHandler.ReconnectRequested += OnTransactionsReconnectRequested;
TransactionsConnectionHandler.Initialize(null);
}
/// <summary>
/// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
/// </summary>
public CoinApiDataQueueHandler(IDataAggregator dataAggregator)
{
_client = new CoinApiWsClient();
_client.TradeEvent += OnTrade;
_client.QuoteEvent += OnQuote;
_client.Error += OnError;
_dataAggregator = dataAggregator;
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
public BinanceBrokerage(string apiKey, string apiSecret, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
: base(WebSocketBaseUrl, new WebSocketClientWrapper(), null, apiKey, apiSecret, "Binance")
{
_job = job;
_algorithm = algorithm;
_aggregator = aggregator;
_messageHandler = new BrokerageConcurrentMessageHandler <WebSocketMessage>(OnMessageImpl);
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
_apiClient = new BinanceRestApiClient(
_symbolMapper,
algorithm?.Portfolio,
apiKey,
apiSecret);
_apiClient.OrderSubmit += (s, e) => OnOrderSubmit(e);
_apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
_apiClient.Message += (s, e) => OnMessage(e);
// User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
_keepAliveTimer = new Timer
{
// 30 minutes
Interval = 30 * 60 * 1000
};
_keepAliveTimer.Elapsed += (s, e) => _apiClient.SessionKeepAlive();
WebSocket.Open += (s, e) => { _keepAliveTimer.Start(); };
WebSocket.Closed += (s, e) => { _keepAliveTimer.Stop(); };
// A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
_reconnectTimer = new Timer
{
// 23.5 hours
Interval = 23.5 * 60 * 60 * 1000
};
_reconnectTimer.Elapsed += (s, e) =>
{
Log.Trace("Daily websocket restart: disconnect");
Disconnect();
Log.Trace("Daily websocket restart: connect");
Connect();
};
}
/// <summary>
/// Constructs instance of <see cref="FrequencyCalculationService"/>.
/// </summary>
/// <param name="dataReader">Instance of <see cref="IDataReader"/> to read input data.</param>
/// <param name="dataWriter">Instance of <see cref="IDataWriter"/> to write output data.</param>
/// <param name="dataAggregator">Instance of <see cref="IDataAggregator"/> to aggregate results.</param>
/// <param name="frequencyCalculator">Instance of <see cref="IFrequencyCalculator"/> to calculate frequencies.</param>
public SingleThreadedFrequencyCalculationService(
IDataReader dataReader,
IDataWriter dataWriter,
IDataAggregator dataAggregator,
IFrequencyCalculator frequencyCalculator)
{
this.dataReader = dataReader;
this.dataWriter = dataWriter;
this.dataAggregator = dataAggregator;
this.frequencyCalculator = frequencyCalculator;
}
/// <summary>
/// IQFeedDataQueueHandler is an implementation of IDataQueueHandler:
/// </summary>
public IQFeedDataQueueHandler(IDataAggregator aggregator)
{
_symbols = new HashSet <Symbol>();
_underlyings = new Dictionary <Symbol, Symbol>();
_aggregator = aggregator;
if (!IsConnected)
{
Connect();
}
}
public TestGDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey,
string apiSecret,
string passPhrase,
IAlgorithm algorithm,
IPriceProvider priceProvider,
IDataAggregator aggregator,
LiveNodePacket job
)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, job)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="OandaRestApiBase"/> class.
/// </summary>
/// <param name="symbolMapper">The symbol mapper.</param>
/// <param name="orderProvider">The order provider.</param>
/// <param name="securityProvider">The holdings provider.</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="environment">The Oanda environment (Trade or Practice)</param>
/// <param name="accessToken">The Oanda access token (can be the user's personal access token or the access token obtained with OAuth by QC on behalf of the user)</param>
/// <param name="accountId">The account identifier.</param>
/// <param name="agent">The Oanda agent string</param>
protected OandaRestApiBase(OandaSymbolMapper symbolMapper, IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, Environment environment, string accessToken, string accountId, string agent)
: base("Oanda Brokerage")
{
SymbolMapper = symbolMapper;
OrderProvider = orderProvider;
SecurityProvider = securityProvider;
Environment = environment;
AccessToken = accessToken;
AccountId = accountId;
Agent = agent;
Aggregator = aggregator;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Refresh();
_subscriptionManager.UnsubscribeImpl += (s, t) => Refresh();
PricingConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
PricingConnectionHandler.ConnectionLost += OnPricingConnectionLost;
PricingConnectionHandler.ConnectionRestored += OnPricingConnectionRestored;
PricingConnectionHandler.ReconnectRequested += OnPricingReconnectRequested;
PricingConnectionHandler.Initialize(null);
TransactionsConnectionHandler = new DefaultConnectionHandler {
MaximumIdleTimeSpan = TimeSpan.FromSeconds(20)
};
TransactionsConnectionHandler.ConnectionLost += OnTransactionsConnectionLost;
TransactionsConnectionHandler.ConnectionRestored += OnTransactionsConnectionRestored;
TransactionsConnectionHandler.ReconnectRequested += OnTransactionsReconnectRequested;
TransactionsConnectionHandler.Initialize(null);
Task.Factory.StartNew(
() =>
{
do
{
_refreshEvent.WaitOne();
Thread.Sleep(SubscribeDelay);
if (!_isConnected)
{
continue;
}
_refreshEvent.Reset();
var symbolsToSubscribe = SubscribedSymbols;
// restart streaming session
SubscribeSymbols(symbolsToSubscribe);
} while (!_streamingCancellationTokenSource.IsCancellationRequested);
},
TaskCreationOptions.LongRunning
);
}
public IngestionProcessor(IPersistentDataProvider persistence, IDataAggregator aggregator, IModelProvider model, IDataMessenger messenger, IRangeMerger <AggregatedDataRange> aggregatedMerger, IRangeMerger <TimeRange> timeMerger, IRangeFilterer <AggregatedDataRange> filterer, SeriesConfiguration configuration, ILogger <IngestionProcessor> logger)
{
_persistence = persistence;
_aggregator = aggregator;
_model = model;
_messenger = messenger;
_aggregatedMerger = aggregatedMerger;
_timeMerger = timeMerger;
_filterer = filterer;
_configuration = configuration;
_logger = logger;
_telemetry.InstrumentationKey = "0def8f5e-9482-48ec-880d-4d2a81834a49";
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">consolidate ticks</param>
public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator)
: base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
{
SubscriptionManager = new BitfinexSubscriptionManager(this, WebSocketUrl, _symbolMapper);
_symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
_algorithm = algorithm;
_aggregator = aggregator;
WebSocket.Open += (sender, args) =>
{
SubscribeAuth();
};
}
public CassandraPiSimDataProvider(CassandraConfiguration configuration, IDataAggregator aggregator, ILogger <CassandraPiSimDataProvider> logger, IRangeMerger <RawDataRange> rawMerger)
{
_configuration = configuration;
_aggregator = aggregator;
_logger = logger;
_rawMerger = rawMerger;
var cluster = Cluster.Builder()
.AddContactPoints(configuration.Address)
.WithDefaultKeyspace(configuration.Keyspace)
.WithQueryTimeout(configuration.QueryTimeout)
.Build();
_session = cluster.Connect();
}
/// <summary>
/// Initializes a new instance of the <see cref="AlpacaBrokerage"/> class.
/// </summary>
/// <param name="orderProvider">The order provider.</param>
/// <param name="securityProvider">The holdings provider.</param>
/// <param name="accountKeyId">The Alpaca api key id</param>
/// <param name="secretKey">The api secret key</param>
/// <param name="tradingMode">The Alpaca trading mode. paper/live</param>
/// <param name="handlesMarketData">true if market data subscriptions will be handled by Alpaca</param>
/// <param name="aggregator">consolidate ticks</param>
public AlpacaBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, string accountKeyId, string secretKey, string tradingMode, bool handlesMarketData, IDataAggregator aggregator)
: base("Alpaca Brokerage")
{
_handlesMarketData = handlesMarketData;
_aggregator = aggregator;
var httpScheme = "https://";
var alpacaBaseUrl = "api.alpaca.markets";
if (tradingMode.Equals("paper"))
{
alpacaBaseUrl = "paper-" + alpacaBaseUrl;
}
var httpAlpacaBaseUrl = httpScheme + alpacaBaseUrl;
_orderProvider = orderProvider;
_securityProvider = securityProvider;
_marketHours = MarketHoursDatabase.FromDataFolder();
// Alpaca trading client
_alpacaTradingClient = new AlpacaTradingClient(new AlpacaTradingClientConfiguration
{
ApiEndpoint = tradingMode.Equals("paper") ? Environments.Paper.AlpacaTradingApi : Environments.Live.AlpacaTradingApi,
SecurityId = new SecretKey(accountKeyId, secretKey)
});
// api client for alpaca data
_polygonDataClient = new PolygonDataClient(new PolygonDataClientConfiguration
{
ApiEndpoint = Environments.Live.PolygonDataApi,
KeyId = accountKeyId
});
// websocket client for alpaca
_sockClient = new SockClient(accountKeyId, secretKey, httpAlpacaBaseUrl);
_sockClient.OnTradeUpdate += OnTradeUpdate;
_sockClient.OnError += OnSockClientError;
// Polygon Streaming client for Alpaca (streams trade and quote data)
_polygonStreamingClient = new PolygonStreamingClient(new PolygonStreamingClientConfiguration
{
ApiEndpoint = Environments.Live.PolygonStreamingApi,
KeyId = accountKeyId,
WebSocketFactory = new WebSocketClientFactory()
});
_polygonStreamingClient.QuoteReceived += OnQuoteReceived;
_polygonStreamingClient.TradeReceived += OnTradeReceived;
_polygonStreamingClient.OnError += OnPolygonStreamingClientError;
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="wssUrl">websockets url</param>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="passPhrase">pass phrase</param>
/// <param name="algorithm">the algorithm instance is required to retreive account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">consolidate ticks</param>
public GDAXBrokerage(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, "GDAX")
{
FillSplit = new ConcurrentDictionary <long, GDAXFill>();
_passPhrase = passPhrase;
_algorithm = algorithm;
_priceProvider = priceProvider;
_aggregator = aggregator;
_isDataQueueHandler = this is GDAXDataQueueHandler;
_fillMonitorTask = Task.Factory.StartNew(FillMonitorAction, _ctsFillMonitor.Token);
}
/// <summary>
/// Initializes a new instance of the <see cref="GDAXDataQueueHandler"/> class
/// </summary>
public GDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator)
{
var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
subscriptionManager.SubscribeImpl += (s, t) =>
{
Subscribe(s);
return(true);
};
subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
SubscriptionManager = subscriptionManager;
}
public IEXDataQueueHandler(IDataAggregator aggregator, bool live)
{
_aggregator = aggregator;
Endpoint = "https://ws-api.iextrading.com/1.0/tops";
if (string.IsNullOrWhiteSpace(_apiKey))
{
Log.Trace("IEXDataQueueHandler(): The IEX API key was not provided, history calls will return no data.");
}
if (live)
{
Reconnect();
}
}
/// <summary>
/// Creates a new instance of the <see cref="FxcmBrokerage"/> class
/// </summary>
/// <param name="orderProvider">The order provider</param>
/// <param name="securityProvider">The holdings provider</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="server">The url of the server</param>
/// <param name="terminal">The terminal name</param>
/// <param name="userName">The user name (login id)</param>
/// <param name="password">The user password</param>
/// <param name="accountId">The account id</param>
public FxcmBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, string server, string terminal, string userName, string password, string accountId)
: base("FXCM Brokerage")
{
_orderProvider = orderProvider;
_securityProvider = securityProvider;
_aggregator = aggregator;
_server = server;
_terminal = terminal;
_userName = userName;
_password = password;
_accountId = accountId;
HistoryResponseTimeout = 5000;
MaximumHistoryRetryAttempts = 1;
}
/// <summary>
/// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
/// </summary>
public FakeDataQueue(IDataAggregator dataAggregator)
{
_aggregator = dataAggregator;
_dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), true);
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"), false);
_optionChainProvider = new LiveOptionChainProvider(_dataCacheProvider, mapFileProvider);
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
_symbolExchangeTimeZones = new Dictionary <Symbol, TimeZoneOffsetProvider>();
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => true;
_subscriptionManager.UnsubscribeImpl += (s, t) => true;
// load it up to start
PopulateQueue();
PopulateQueue();
PopulateQueue();
PopulateQueue();
_timer = new Timer
{
AutoReset = false,
Enabled = true,
Interval = 1000,
};
var lastCount = 0;
var lastTime = DateTime.UtcNow;
_timer.Elapsed += (sender, args) =>
{
var elapsed = (DateTime.UtcNow - lastTime);
var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
lastCount = _count;
lastTime = DateTime.UtcNow;
PopulateQueue();
try
{
_timer.Reset();
}
catch (ObjectDisposedException)
{
// pass
}
};
}
/// <summary>
/// Initializes a new instance of the <see cref="GDAXDataQueueHandler"/> class
/// </summary>
public GDAXDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator, LiveNodePacket job)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, job)
{
Initialize(
wssUrl: wssUrl,
websocket: websocket,
restClient: restClient,
apiKey: apiKey,
apiSecret: apiSecret,
passPhrase: passPhrase,
algorithm: algorithm,
priceProvider: priceProvider,
aggregator: aggregator,
job: job
);
}
public AnomalyDetection(IOrchestrator orcherstratorProxy, IDataAggregator aggregatorProxy)
{
orcherstratorProxy.Detection.Subscribe(this, async signal =>
{
int cluster = 0;
if (_kMeansClustering != null)
{
lock (_syncClustering)
if (_kMeansClustering != null)
{
cluster = _kMeansClustering.Classify(new double[] { signal.Value });
}
}
if (cluster < 0)
{
System.Console.WriteLine("_____________________________Anomaly detected__________________________________");
await Anomaly.PublishAsync(new Anomaly()
{
Temperature = signal
});
}
return(MessageResult.Ok);
});
aggregatorProxy.Aggregate.Subscribe(this, async(sampleReference) =>
{
//if the messages has been stored and forwarded, but the file has been deleted (e.g. a restart)
//then the message can be empty (null)
if (sampleReference == null)
{
return(MessageResult.Ok);
}
System.Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
lock (_syncSample)
_sample = sampleReference.Message.Values;
lock (_syncClustering)
_kMeansClustering = new KMeansClustering(_sample, _numClusters);
return(MessageResult.Ok);
});
}
/// <summary>
/// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
/// </summary>
public FakeDataQueue(IDataAggregator dataAggregator)
{
_aggregator = dataAggregator;
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
_symbolExchangeTimeZones = new Dictionary <Symbol, TimeZoneOffsetProvider>();
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => true;
_subscriptionManager.UnsubscribeImpl += (s, t) => true;
// load it up to start
PopulateQueue();
PopulateQueue();
PopulateQueue();
PopulateQueue();
_timer = new Timer
{
AutoReset = false,
Enabled = true,
Interval = 1000,
};
var lastCount = 0;
var lastTime = DateTime.UtcNow;
_timer.Elapsed += (sender, args) =>
{
var elapsed = (DateTime.UtcNow - lastTime);
var ticksPerSecond = (_count - lastCount) / elapsed.TotalSeconds;
Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
lastCount = _count;
lastTime = DateTime.UtcNow;
PopulateQueue();
try
{
_timer.Reset();
}
catch (ObjectDisposedException)
{
// pass
}
};
}
/// <summary>
/// Creates a new instance of the <see cref="FxcmBrokerage"/> class
/// </summary>
/// <param name="orderProvider">The order provider</param>
/// <param name="securityProvider">The holdings provider</param>
/// <param name="aggregator">Consolidate ticks</param>
/// <param name="server">The url of the server</param>
/// <param name="terminal">The terminal name</param>
/// <param name="userName">The user name (login id)</param>
/// <param name="password">The user password</param>
/// <param name="accountId">The account id</param>
public FxcmBrokerage(IOrderProvider orderProvider, ISecurityProvider securityProvider, IDataAggregator aggregator, string server, string terminal, string userName, string password, string accountId)
: base("FXCM Brokerage")
{
_orderProvider = orderProvider;
_securityProvider = securityProvider;
_aggregator = aggregator;
_server = server;
_terminal = terminal;
_userName = userName;
_password = password;
_accountId = accountId;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
HistoryResponseTimeout = 5000;
MaximumHistoryRetryAttempts = 1;
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="websocket">instance of websockets client</param>
/// <param name="restClient">instance of rest client</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="priceProvider">The price provider for missing FX conversion rates</param>
/// <param name="aggregator">consolidate ticks</param>
public BitfinexBrokerage(IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider, IDataAggregator aggregator)
: base(WebSocketUrl, websocket, restClient, apiKey, apiSecret, "Bitfinex")
{
SubscriptionManager = new BitfinexSubscriptionManager(this, WebSocketUrl, _symbolMapper);
_symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
_algorithm = algorithm;
_aggregator = aggregator;
// load currency map
using (var wc = new WebClient())
{
var json = wc.DownloadString("https://api-pub.bitfinex.com/v2/conf/pub:map:currency:sym");
var rows = JsonConvert.DeserializeObject <List <List <List <string> > > >(json)[0];
_currencyMap = rows
.ToDictionary(row => row[0], row => row[1].ToUpperInvariant());
}
WebSocket.Open += (sender, args) =>
{
SubscribeAuth();
};
}
public Level1Port(IDataAggregator aggregator, IQFeedDataQueueUniverseProvider symbolUniverse)
: base(80)
{
start = DateTime.Now;
_prices = new ConcurrentDictionary <string, double>();
_openInterests = new ConcurrentDictionary <string, int>();
_aggregator = aggregator;
_symbolUniverse = symbolUniverse;
Level1SummaryUpdateEvent += OnLevel1SummaryUpdateEvent;
Level1TimerEvent += OnLevel1TimerEvent;
Level1ServerDisconnectedEvent += OnLevel1ServerDisconnected;
Level1ServerReconnectFailed += OnLevel1ServerReconnectFailed;
Level1UnknownEvent += OnLevel1UnknownEvent;
Level1FundamentalEvent += OnLevel1FundamentalEvent;
_timer = new Timer(1000);
_timer.Enabled = false;
_timer.AutoReset = true;
_timer.Elapsed += (sender, args) =>
{
var ticksPerSecond = count / (DateTime.Now - start).TotalSeconds;
int dataQueueCount = Interlocked.Exchange(ref _dataQueueCount, 0);
if (ticksPerSecond > 1000 || dataQueueCount > 31)
{
Log.Trace($"IQFeed.OnSecond(): Ticks/sec: {ticksPerSecond.ToStringInvariant("0000.00")} " +
$"Engine.Ticks.Count: {dataQueueCount} CPU%: {OS.CpuUsage.ToStringInvariant("0.0") + "%"}"
);
}
count = 0;
start = DateTime.Now;
};
_timer.Enabled = true;
}
/// <summary>
/// Initializes a new instance of the <see cref="CoinApiDataQueueHandler"/> class
/// </summary>
public CoinApiDataQueueHandler()
{
_dataAggregator = Composer.Instance.GetPart <IDataAggregator>();
if (_dataAggregator == null)
{
_dataAggregator =
Composer.Instance.GetExportedValueByTypeName <IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
}
var product = Config.GetValue <CoinApiProduct>("coinapi-product");
_streamingDataType = product < CoinApiProduct.Streamer
? new[] { "trade" }
: new[] { "trade", "quote" };
Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'");
_client = new CoinApiWsClient();
_client.TradeEvent += OnTrade;
_client.QuoteEvent += OnQuote;
_client.Error += OnError;
_subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
_subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s);
_subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s);
}
public GDAXFakeDataQueueHandler(string wssUrl, IWebSocket websocket, IRestClient restClient, string apiKey, string apiSecret, string passPhrase, IAlgorithm algorithm,
IPriceProvider priceProvider, IDataAggregator aggregator)
: base(wssUrl, websocket, restClient, apiKey, apiSecret, passPhrase, algorithm, priceProvider, aggregator, null)
{
}
/// <summary>
/// Constructor for brokerage
/// </summary>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="restApiUrl">The rest api url</param>
/// <param name="webSocketBaseUrl">The web socket base url</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
public BinanceBrokerage(string apiKey, string apiSecret, string restApiUrl, string webSocketBaseUrl, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
: base("Binance")
{
Initialize(
wssUrl: webSocketBaseUrl,
restApiUrl: restApiUrl,
apiKey: apiKey,
apiSecret: apiSecret,
algorithm: algorithm,
aggregator: aggregator,
job: job
);
}
/// <summary>
/// Initialize the instance of this class
/// </summary>
/// <param name="wssUrl">The web socket base url</param>
/// <param name="restApiUrl">The rest api url</param>
/// <param name="apiKey">api key</param>
/// <param name="apiSecret">api secret</param>
/// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
/// <param name="aggregator">the aggregator for consolidating ticks</param>
/// <param name="job">The live job packet</param>
private void Initialize(string wssUrl, string restApiUrl, string apiKey, string apiSecret,
IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
{
if (IsInitialized)
{
return;
}
base.Initialize(wssUrl, new WebSocketClientWrapper(), null, apiKey, apiSecret);
_job = job;
_algorithm = algorithm;
_aggregator = aggregator;
_webSocketBaseUrl = wssUrl;
_messageHandler = new BrokerageConcurrentMessageHandler <WebSocketMessage>(OnUserMessage);
var maximumWebSocketConnections = Config.GetInt("binance-maximum-websocket-connections");
var symbolWeights = maximumWebSocketConnections > 0 ? FetchSymbolWeights() : null;
var subscriptionManager = new BrokerageMultiWebSocketSubscriptionManager(
wssUrl,
MaximumSymbolsPerConnection,
maximumWebSocketConnections,
symbolWeights,
() => new BinanceWebSocketWrapper(null),
Subscribe,
Unsubscribe,
OnDataMessage,
new TimeSpan(23, 45, 0));
SubscriptionManager = subscriptionManager;
// can be null, if BinanceBrokerage is used as DataQueueHandler only
if (_algorithm != null)
{
// Binance rest api endpoint is different for sport and margin trading
// we need to delay initialization of rest api client until Algorithm is initialized
// and user brokerage choise is actually applied
_apiClientLazy = new Lazy <BinanceBaseRestApiClient>(() =>
{
BinanceBaseRestApiClient apiClient = _algorithm.BrokerageModel.AccountType == AccountType.Cash
? new BinanceSpotRestApiClient(_symbolMapper, algorithm?.Portfolio, apiKey, apiSecret, restApiUrl)
: new BinanceCrossMarginRestApiClient(_symbolMapper, algorithm?.Portfolio, apiKey, apiSecret,
restApiUrl);
apiClient.OrderSubmit += (s, e) => OnOrderSubmit(e);
apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
apiClient.Message += (s, e) => OnMessage(e);
// once we know the api endpoint we can subscribe to user data stream
apiClient.CreateListenKey();
_keepAliveTimer.Elapsed += (s, e) => apiClient.SessionKeepAlive();
Connect(apiClient.SessionId);
return(apiClient);
});
}
// User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
_keepAliveTimer = new Timer
{
// 30 minutes
Interval = 30 * 60 * 1000
};
WebSocket.Open += (s, e) =>
{
_keepAliveTimer.Start();
};
WebSocket.Closed += (s, e) =>
{
ApiClient.StopSession();
_keepAliveTimer.Stop();
};
// A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
// Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
_reconnectTimer = new Timer
{
// 23.5 hours
Interval = 23.5 * 60 * 60 * 1000
};
_reconnectTimer.Elapsed += (s, e) =>
{
Log.Trace("Daily websocket restart: disconnect");
Disconnect();
Log.Trace("Daily websocket restart: connect");
Connect();
};
}