public SymbolFeed(BinanceClient client, CombinedWebSocketClient websocket, BinanceTradeBarsRepository hist, string market, SymbolInfo symbol, DateTime timeNow)
{
HistoryDb = hist;
this.Client = client;
this.WebSocketClient = websocket;
this.Symbol = symbol;
this.Market = market;
this.Time = timeNow;
Logger = LogManager.GetLogger("Bin" + Symbol + "Feed");
HistoryId = new SymbolHistoryId(this.Market, Symbol.Key, TimeSpan.FromSeconds(60));
}
private void InitializeDataSourceCallBack(Market market, SymbolFeed feed)
{
if (!IncrementalHistoryLoading)
{
if (feed.DataSource == null)
{
var histInfo = new SymbolHistoryId(market.MarketName, feed.Symbol.Key, TimeSpan.FromSeconds(60));
feed.DataSource = this.HistoryDb.GetSymbolHistory(histInfo, StartTime, EndTime);
//this.HistoryDb.CloseFile(histInfo);
}
}
}
private async Task CheckClosing()
{
while (true)
{
lock (LastHistoryRequest)
{
foreach (var kv in LastHistoryRequest.ToArray())
{
var key = kv.Key;
var lastReq = kv.Value;
if (DateTime.Now > lastReq.AddMinutes(5))
{
this.SaveAndClose(SymbolHistoryId.Parse(key), true);
LastHistoryRequest.Remove(key);
}
}
}
await Task.Delay(10000);
}
}
public async Task AssureFilter(Func <string, bool> filter, DateTime fromTime, DateTime toTime)
{
var exchangeInfo = Client.GetExchangeInfo().Result;
var symbols = exchangeInfo.Symbols;
var toDownload = symbols
.Where(s => filter(s.symbol))
.Select(sp => sp.symbol).ToList();
toDownload.Sort();
List <Task> tasks = new List <Task>();
foreach (var sym in toDownload)
{
var histInfo = new SymbolHistoryId("Binance", sym, TimeSpan.FromMinutes(1));
var task = this.AssureData(histInfo, fromTime, toTime).ContinueWith(t => this.SaveAndClose(histInfo, true));
tasks.Add(task);
}
await Task.WhenAll(tasks);
}
public bool NextTick()
{
//calculate next tick time
var nextTick = this.Time + Resolution;
//if a new month has started
//update market time
this.Time = nextTick;
bool moreData = false;
//add new data to all symbol feeds that have it
foreach (var market in _Markets)
{
if (IncrementalHistoryLoading)
{
if (Time.Month != market.Time.Month || Time.Year != market.Time.Year || !market.FistTickPassed)
{
// a new month has started, let's load the data for this month
foreach (var feed in market.SymbolsFeeds.Values)
{
var histInfo = new SymbolHistoryId(market.MarketName, feed.Symbol.Key, TimeSpan.FromSeconds(60));
feed.DataSource = this.HistoryDb.GetSymbolHistory(
histInfo,
this.Time,
new DateTime(Time.Year, Time.Month, 1, 0, 0, 0).AddMonths(1).AddMilliseconds(-1));
this.HistoryDb.SaveAndClose(histInfo, false);
market.FistTickPassed = true;
}
}
}
market.Time = this.Time;
foreach (var feed in market.SymbolsFeeds.Values)
{
feed.Time = market.Time;
var dataSource = feed.DataSource;
if (dataSource.Ticks.Count > 0)
{
while (dataSource.Ticks.NextTickTime <= this.Time)
{
dataSource.Ticks.MoveNext();
var candle = dataSource.Ticks.Current is Candlestick c ? c : new Candlestick(dataSource.Ticks.Current);
market.AddNewCandle(feed as SymbolFeed, candle);// new Candlestick(data.Ticks.Tick)); //use less memory
}
moreData |= dataSource.Ticks.Position < dataSource.Ticks.Count - 1;
}
}
}
if (!moreData)
{
NoMoreDataCount++;
}
else
{
NoMoreDataCount = 0;
}
//resolve orders of each market
foreach (var market in _Markets)
{
market.ResolveOrders();
}
//raise orders/trades events
foreach (var market in _Markets)
{
market.RaisePendingEvents();
}
return(nextTick <= this.EndTime && NoMoreDataCount < 10);
}
public async Task AssureData(SymbolHistoryId histInfo, DateTime fromTime, DateTime toTime)
{
if (toTime > DateTime.Now.AddYears(10))
{
toTime = DateTime.Now.AddYears(10);
}
toTime = new DateTime(toTime.Year, toTime.Month, toTime.Day, toTime.Hour, toTime.Minute, 0, DateTimeKind.Utc);
var epoch = new DateTime(2017, 07, 01, 0, 0, 0, DateTimeKind.Utc);
if (fromTime < epoch)
{
fromTime = epoch;
}
await DownloadCandlesSemaphore.WaitAsync();
var sem = GetSemaphore(histInfo.Symbol);
try
{
await sem.WaitAsync();
var hist = this.GetSymbolHistory(histInfo, fromTime, toTime);
var oldTicks = hist.Ticks;
//first find next available data, if not found download everything
DateTime lastTime = fromTime;
while (lastTime < toTime)
{
if (oldTicks.MoveNext())
{
if (oldTicks.Time - lastTime > histInfo.Resolution)
{
Logger.Debug($"Hole found in {histInfo.Symbol} history from {lastTime} to {hist.Ticks.NextTickTime}.");
var candles = await DownloadCandles(histInfo.Symbol, lastTime, oldTicks.Current.CloseTime);
this.AddCandlesticks(histInfo, candles);
}
lastTime = oldTicks.Current.CloseTime;
}
else
{
Logger.Debug($"Hole found in {histInfo.Symbol} history from {lastTime} to {toTime}.");
//there isn't any other tick, all remaining data needs to be downloaded
var candles = await DownloadCandles(histInfo.Symbol, lastTime, toTime);
this.AddCandlesticks(histInfo, candles);
lastTime = toTime;
}
}
}
catch (Exception ex)
{
Logger.Error("Fatal Exception while download history for symbol {0}: {1}", histInfo.Symbol, ex.Message);
}
finally
{
DownloadCandlesSemaphore.Release();
sem.Release();
}
//this.SaveAll();
}
public async Task DownloadHistoryAsync(string symbol, DateTime fromTime, TimeSpan redownloadStart)
{
//we must assure that there is only one downloading action ongoing for each symbol!
var semaphore = GetSemaphore(symbol);
try
{
await semaphore.WaitAsync();
Console.WriteLine($"Downloading {symbol} history ");
DateTime endTime = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(60));
//we need to convert all in UTC time
var startTime = fromTime; //
var epoch = new DateTime(2017, 07, 01, 0, 0, 0, DateTimeKind.Utc);
if (startTime < epoch)
{
startTime = epoch;
}
//try to get the first candle - use these functions for performance improvement
var symId = new SymbolHistoryId(MarketName, symbol, TimeSpan.FromSeconds(60));
var metaData = this.GetMetaData(symId);
if (metaData.FirstBar != null && metaData.LastBar != null)
{
if (startTime > metaData.FirstBar.OpenTime)
{
//if startTime is after the first recorded bar then
// then we can start downloading from last recorded bar
if (endTime <= metaData.LastBar.Time)
{
endTime = startTime;
}
startTime = new DateTime(metaData.LastBar.OpenTime.Ticks, DateTimeKind.Utc).Subtract(redownloadStart);
}
else
{
if (metaData.FirstKnownData == null)
{
//start time is earlier than first known candle.. we start download from first candle on server
var firstAvailable = (await Client.GetKlinesCandlesticks(
new GetKlinesCandlesticksRequest
{
Symbol = symbol,
StartTime = startTime,
Interval = KlineInterval.OneMinute,
EndTime = startTime.AddYears(3),
Limit = 10
})).FirstOrDefault();
this.UpdateFirstKnownData(metaData.HistoryId, KlineToCandlestick(firstAvailable));
}
//if we already downloaded the first available we can start download from the last known
//otherwise we leave start time as it is to avoid creating holes
if (metaData.FirstKnownData.CloseTime.AddSeconds(1) >= metaData.FirstBar.OpenTime)
{
startTime = new DateTime(metaData.LastBar.Time.Ticks, DateTimeKind.Utc).Subtract(redownloadStart);
}
// avoid downloading again some data if possible
if (endTime <= metaData.LastBar.Time)
{
endTime = metaData.FirstBar.Time;
}
}
}
//download and add bars if needed
if (endTime > startTime)
{
var candlesDownloaded = await DownloadCandles(symbol, startTime, endTime);
//---
this.AddCandlesticks(symId, candlesDownloaded);
this.ValidateData(symId);
this.SaveAndClose(symId);
}
Console.WriteLine($"{symbol} history downloaded");
}
catch (Exception ex)
{
var msg = $"Fatal Exception during {symbol} history download: ";
if (ex is BinanceException binException)
{
msg += binException.ErrorDetails;
}
else
{
msg += ex.Message;
}
Console.WriteLine(msg);
}
finally
{
semaphore.Release();
}
}
public override ISymbolHistory GetSymbolHistory(SymbolHistoryId info, DateTime startOfData, DateTime endOfData)
{
lock (LastHistoryRequest)
LastHistoryRequest[info.Key] = DateTime.Now;
return(base.GetSymbolHistory(info, startOfData, endOfData));
}
