private async void EnsureHistoricalTrades()
{
if (!Exchange.SupportsHistoricalLoad)
{
return;
}
Logger.LogInformation("Ensure historical trades are captured.");
foreach (var symbolCode in ExchangeWorker.Configuration.Symbol)
{
var symbol = SymbolFactory.Get(symbolCode);
var lastTradeFilter = await HistorianRepository.GetTradeFilter(Exchange.Name, symbolCode);
var priority = string.Equals(lastTradeFilter, Exchange.GetHttpClient().InitialTradeFilter) ? 2 : 1;
var catchup = new HistorianTradeCatchup
{
Exchange = Exchange.Name,
SymbolCode = symbolCode,
TradeFilter = lastTradeFilter,
EpochTo = Epoch.Now,
CurrentTradeFilter = lastTradeFilter,
Priority = priority
};
await HistorianRepository.AddTradeCatchup(catchup);
}
}
private void BeginReceiveTradesWebSocket() => Task.Run(() =>
{
ExchangeTradeCatchupWorker.Start(ExchangeWorker, Limit);
var symbols = ExchangeWorker.Configuration.Symbol.Select(s => SymbolFactory.Get(s)).ToList();
using (Logger.BeginProtocolScope("Web Socket"))
{
Logger.LogInformation($"Establishing connection");
WebSocketClient.OnOpen += delegate(object sender, EventArgs e)
{
Logger.LogInformation($"Established connection");
Logger.LogInformation($"Begin listening for trades");
if (WebSocketClient.IsSubscribeModel)
{
WebSocketClient.BeginListenTrades(symbols);
}
};
WebSocketClient.OnClose += delegate(object sender, CloseEventArgs e)
{
if (ExchangeWorker.Online)
{
Logger.LogWarning($"Connection has been closed");
}
RetryWebSocketConnect();
};
WebSocketClient.OnTradesReceived += async delegate(object sender, TradesReceivedEventArgs e)
{
using (Logger.BeginSymbolScope(e.Data.SymbolCode))
{
Logger.LogInformation($"Trades received");
using (var transaction = await StorageTransactionFactory.Begin())
{
await ExchangeTradeProvider.AddTrades(transaction, Logger, e.Data);
await HistorianRepository.SetTradeFilter(transaction, Exchange.Name, e.Data.SymbolCode, e.Data.Filter);
await transaction.Commit();
}
}
};
WebSocketClient.Begin();
if (!WebSocketClient.IsSubscribeModel)
{
WebSocketClient.BeginListenTrades(symbols);
}
}
});
private void AggregateTradeStats(ISymbol baseSymbol, ExchangeStatsKeyEnum statsKey, int processLimit) => Task.Run(async() =>
{
var fullSymbolCode = ExchangeTradeStatProvider.Convert(baseSymbol.Code, statsKey);
var logger = LoggerFactory.CreateLogger($"Historian.{Exchange.Name}.{fullSymbolCode}.Worker.TradeStatAggregate");
using (logger.BeginExchangeScope(Exchange.Name))
{
using (logger.BeginSymbolScope(fullSymbolCode))
{
using (logger.BeginExchangeStatsScope(statsKey))
{
var tradeStatId = (await HistorianRepository.GetLastTradeStatId(Exchange.Name, fullSymbolCode)).GetValueOrDefault(0);
while (true)
{
try
{
var s = DateTime.Now;
var tradeStats = await MarketRepository.GetNextTradeStats(Exchange.Name, baseSymbol.Code, statsKey, tradeStatId, processLimit);
if (tradeStats.Count > 0)
{
using (var transaction = await StorageTransactionFactory.Begin())
{
await MarketRepository.SaveTradeStatAggregates(transaction, Exchange.Name, fullSymbolCode, tradeStats);
tradeStatId = tradeStats.Max(t => t.TradeStatId);
await HistorianRepository.SetLastTradeStatId(transaction, Exchange.Name, fullSymbolCode, tradeStatId);
await transaction.Commit();
}
var e = DateTime.Now;
logger.LogInformation($"Aggregation up to trade stat id {tradeStatId} took {(e.Subtract(s).TotalMilliseconds)}ms.");
}
await Task.Delay(5);
}
catch (Exception ex)
{
logger.LogCritical(ex, "Aggregation failed.");
await Task.Delay(250);
}
}
}
}
}
});
private void BeginReceiveTradesHttp() => Task.Run(async() =>
{
using (Logger.BeginProtocolScope("Https"))
{
var symbols = ExchangeWorker.Configuration.Symbol.Select(symbolCode => SymbolFactory.Get(symbolCode)).Where(symbol => symbol.Tradable);
while (true)
{
if (!ExchangeWorker.Online)
{
await Task.Delay(1000);
continue;
}
foreach (var symbol in symbols)
{
using (Logger.BeginSymbolScope(symbol.Code))
{
try
{
var lastTradeFilter = await HistorianRepository.GetTradeFilter(Exchange.Name, symbol.Code);
TradeResult result;
using (var transaction = await StorageTransactionFactory.Begin())
{
result = await ExchangeTradeProvider.ReceiveTradesHttp(transaction, Logger, Exchange.Name, symbol, HttpClient, Limit, lastTradeFilter);
if (result != null)
{
await HistorianRepository.SetTradeFilter(transaction, Exchange.Name, symbol.Code, result.Filter);
}
await transaction.Commit();
}
if (result == null)
{
await Task.Delay(1000);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Unexpected error occurred");
}
}
}
}
}
});
private void BeginProcessQueue() => Task.Run(async() =>
{
Queue = new ConcurrentQueue <HistorianLogItem>();
while (true)
{
if (Queue.Count > 0)
{
HistorianLogItem logItem;
if (Queue.TryDequeue(out logItem))
{
await HistorianRepository.WriteLog(logItem);
}
}
await Task.Delay(25);
}
});
private async Task Ping(bool initial = false)
{
try
{
var ping = new Ping();
var uri = new Uri(Exchange.GetHttpClient().ApiUrl);
var reply = await ping.SendPingAsync(uri.Host);
if (reply.Status == IPStatus.Success)
{
// If the exchange is running web socket, and supports historical loads, ensure the historian does not produce gaps
if (!initial && !Online && Exchange.SupportsHistoricalLoad && Exchange.GetWebSocketClient() != null)
{
Logger.LogInformation($"Regained connectivity");
var symbols = await HistorianRepository.GetSymbols(Exchange.Name);
foreach (var historianSymbol in symbols)
{
var symbol = SymbolFactory.Get(historianSymbol.SymbolCode);
if (!symbol.Tradable)
{
continue;
}
using (Logger.BeginSymbolScope(symbol.Code))
{
var now = Epoch.Now;
var catchup = new HistorianTradeCatchup
{
Exchange = Exchange.Name,
SymbolCode = historianSymbol.SymbolCode,
TradeFilter = historianSymbol.TradeFilter,
EpochTo = now,
CurrentTradeFilter = historianSymbol.TradeFilter,
Priority = 1
};
Logger.LogWarning($"Adding trade catch up job with filter '{historianSymbol.TradeFilter}'");
await HistorianRepository.AddTradeCatchup(catchup);
}
}
}
Online = true;
}
else
{
Logger.LogWarning($"Lost connectivity: {reply.Status}");
Online = false;
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, $"Unexpected error occurred when pinging");
Online = false;
}
}
private void BeginTradeCatchupWorker(int limit) => Task.Run(async() =>
{
using (Logger.BeginProtocolScope("Https"))
{
while (true)
{
foreach (var symbolCode in ExchangeWorker.Configuration.Symbol)
{
using (Logger.BeginSymbolScope(symbolCode))
{
var symbol = SymbolFactory.Get(symbolCode);
if (!symbol.Tradable)
{
continue;
}
try
{
if (!ExchangeWorker.Online)
{
await Task.Delay(2000);
continue;
}
var current = await HistorianRepository.GetNextTradeCatchup(Exchange.Name, symbolCode);
if (current == null)
{
continue;
}
TradeResult result = null;
using (var transaction = await StorageTransactionFactory.Begin())
{
result = await ExchangeTradeProvider.ReceiveTradesHttp(transaction, Logger, Exchange.Name, symbol, HttpClient, limit, current.CurrentTradeFilter);
await transaction.Commit();
}
if (result == null)
{
continue;
}
var lastTrade = result.Trades.LastOrDefault();
if (lastTrade.Epoch.TimestampMilliseconds > current.EpochTo.TimestampMilliseconds)
{
await HistorianRepository.RemoveTradeCatchup(current);
}
else
{
current.CurrentTradeFilter = result.Filter;
await HistorianRepository.UpdateTradeCatchup(current);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Unable to get trade catchup jobs.");
}
}
}
}
}
});
public async Task <bool> Run(ILogger logger)
{
logger.LogInformation("Running repository bootstrapper");
try
{
foreach (var currency in CurrencyFactory.List())
{
await CurrencyRepository.Add(currency);
}
foreach (var symbol in SymbolFactory.List())
{
await SymbolRepository.Add(symbol);
}
foreach (var group in IntervalFactory.ListGroups())
{
foreach (var ik in IntervalFactory.ListIntervalKeys(group.IntervalGroup))
{
await IntervalRepository.Add(ik);
}
}
foreach (var exchange in ExchangeFactory.List())
{
await ExchangeRepository.Add(exchange);
var httpClient = exchange.GetHttpClient();
foreach (var symbolCode in exchange.Symbol)
{
var symbol = SymbolFactory.Get(symbolCode);
await SymbolRepository.Add(symbol);
await ExchangeRepository.AddSymbol(exchange.Name, symbolCode);
var tradeFilter = await HistorianRepository.GetTradeFilter(exchange.Name, symbolCode);
if (tradeFilter == null)
{
using (var transaction = await StorageTransactionFactory.Begin())
{
await HistorianRepository.SetTradeFilter(transaction, exchange.Name, symbolCode, httpClient.InitialTradeFilter);
await transaction.Commit();
}
}
}
}
foreach (var orderSide in OrderSideFactory.List())
{
await OrderSideRepository.Add(orderSide);
}
return(true);
}
catch (Exception ex)
{
logger.LogCritical(ex, "Unable to run repository bootstrapper");
return(false);
}
}