public SymbolFeed(BinanceClient client, CombinedWebSocketClient websocket, BinanceTradeBarsRepository hist, string market, SymbolInfo symbol, DateTime timeNow) { HistoryDb = hist; this.Client = client; this.WebSocketClient = websocket; this.Symbol = symbol; this.Market = market; this.Time = timeNow; Logger = LogManager.GetLogger("Bin" + Symbol + "Feed"); HistoryId = new SymbolHistoryId(this.Market, Symbol.Key, TimeSpan.FromSeconds(60)); }
private void InitializeDataSourceCallBack(Market market, SymbolFeed feed) { if (!IncrementalHistoryLoading) { if (feed.DataSource == null) { var histInfo = new SymbolHistoryId(market.MarketName, feed.Symbol.Key, TimeSpan.FromSeconds(60)); feed.DataSource = this.HistoryDb.GetSymbolHistory(histInfo, StartTime, EndTime); //this.HistoryDb.CloseFile(histInfo); } } }
private async Task CheckClosing() { while (true) { lock (LastHistoryRequest) { foreach (var kv in LastHistoryRequest.ToArray()) { var key = kv.Key; var lastReq = kv.Value; if (DateTime.Now > lastReq.AddMinutes(5)) { this.SaveAndClose(SymbolHistoryId.Parse(key), true); LastHistoryRequest.Remove(key); } } } await Task.Delay(10000); } }
public async Task AssureFilter(Func <string, bool> filter, DateTime fromTime, DateTime toTime) { var exchangeInfo = Client.GetExchangeInfo().Result; var symbols = exchangeInfo.Symbols; var toDownload = symbols .Where(s => filter(s.symbol)) .Select(sp => sp.symbol).ToList(); toDownload.Sort(); List <Task> tasks = new List <Task>(); foreach (var sym in toDownload) { var histInfo = new SymbolHistoryId("Binance", sym, TimeSpan.FromMinutes(1)); var task = this.AssureData(histInfo, fromTime, toTime).ContinueWith(t => this.SaveAndClose(histInfo, true)); tasks.Add(task); } await Task.WhenAll(tasks); }
public bool NextTick() { //calculate next tick time var nextTick = this.Time + Resolution; //if a new month has started //update market time this.Time = nextTick; bool moreData = false; //add new data to all symbol feeds that have it foreach (var market in _Markets) { if (IncrementalHistoryLoading) { if (Time.Month != market.Time.Month || Time.Year != market.Time.Year || !market.FistTickPassed) { // a new month has started, let's load the data for this month foreach (var feed in market.SymbolsFeeds.Values) { var histInfo = new SymbolHistoryId(market.MarketName, feed.Symbol.Key, TimeSpan.FromSeconds(60)); feed.DataSource = this.HistoryDb.GetSymbolHistory( histInfo, this.Time, new DateTime(Time.Year, Time.Month, 1, 0, 0, 0).AddMonths(1).AddMilliseconds(-1)); this.HistoryDb.SaveAndClose(histInfo, false); market.FistTickPassed = true; } } } market.Time = this.Time; foreach (var feed in market.SymbolsFeeds.Values) { feed.Time = market.Time; var dataSource = feed.DataSource; if (dataSource.Ticks.Count > 0) { while (dataSource.Ticks.NextTickTime <= this.Time) { dataSource.Ticks.MoveNext(); var candle = dataSource.Ticks.Current is Candlestick c ? c : new Candlestick(dataSource.Ticks.Current); market.AddNewCandle(feed as SymbolFeed, candle);// new Candlestick(data.Ticks.Tick)); //use less memory } moreData |= dataSource.Ticks.Position < dataSource.Ticks.Count - 1; } } } if (!moreData) { NoMoreDataCount++; } else { NoMoreDataCount = 0; } //resolve orders of each market foreach (var market in _Markets) { market.ResolveOrders(); } //raise orders/trades events foreach (var market in _Markets) { market.RaisePendingEvents(); } return(nextTick <= this.EndTime && NoMoreDataCount < 10); }
public async Task AssureData(SymbolHistoryId histInfo, DateTime fromTime, DateTime toTime) { if (toTime > DateTime.Now.AddYears(10)) { toTime = DateTime.Now.AddYears(10); } toTime = new DateTime(toTime.Year, toTime.Month, toTime.Day, toTime.Hour, toTime.Minute, 0, DateTimeKind.Utc); var epoch = new DateTime(2017, 07, 01, 0, 0, 0, DateTimeKind.Utc); if (fromTime < epoch) { fromTime = epoch; } await DownloadCandlesSemaphore.WaitAsync(); var sem = GetSemaphore(histInfo.Symbol); try { await sem.WaitAsync(); var hist = this.GetSymbolHistory(histInfo, fromTime, toTime); var oldTicks = hist.Ticks; //first find next available data, if not found download everything DateTime lastTime = fromTime; while (lastTime < toTime) { if (oldTicks.MoveNext()) { if (oldTicks.Time - lastTime > histInfo.Resolution) { Logger.Debug($"Hole found in {histInfo.Symbol} history from {lastTime} to {hist.Ticks.NextTickTime}."); var candles = await DownloadCandles(histInfo.Symbol, lastTime, oldTicks.Current.CloseTime); this.AddCandlesticks(histInfo, candles); } lastTime = oldTicks.Current.CloseTime; } else { Logger.Debug($"Hole found in {histInfo.Symbol} history from {lastTime} to {toTime}."); //there isn't any other tick, all remaining data needs to be downloaded var candles = await DownloadCandles(histInfo.Symbol, lastTime, toTime); this.AddCandlesticks(histInfo, candles); lastTime = toTime; } } } catch (Exception ex) { Logger.Error("Fatal Exception while download history for symbol {0}: {1}", histInfo.Symbol, ex.Message); } finally { DownloadCandlesSemaphore.Release(); sem.Release(); } //this.SaveAll(); }
public async Task DownloadHistoryAsync(string symbol, DateTime fromTime, TimeSpan redownloadStart) { //we must assure that there is only one downloading action ongoing for each symbol! var semaphore = GetSemaphore(symbol); try { await semaphore.WaitAsync(); Console.WriteLine($"Downloading {symbol} history "); DateTime endTime = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(60)); //we need to convert all in UTC time var startTime = fromTime; // var epoch = new DateTime(2017, 07, 01, 0, 0, 0, DateTimeKind.Utc); if (startTime < epoch) { startTime = epoch; } //try to get the first candle - use these functions for performance improvement var symId = new SymbolHistoryId(MarketName, symbol, TimeSpan.FromSeconds(60)); var metaData = this.GetMetaData(symId); if (metaData.FirstBar != null && metaData.LastBar != null) { if (startTime > metaData.FirstBar.OpenTime) { //if startTime is after the first recorded bar then // then we can start downloading from last recorded bar if (endTime <= metaData.LastBar.Time) { endTime = startTime; } startTime = new DateTime(metaData.LastBar.OpenTime.Ticks, DateTimeKind.Utc).Subtract(redownloadStart); } else { if (metaData.FirstKnownData == null) { //start time is earlier than first known candle.. we start download from first candle on server var firstAvailable = (await Client.GetKlinesCandlesticks( new GetKlinesCandlesticksRequest { Symbol = symbol, StartTime = startTime, Interval = KlineInterval.OneMinute, EndTime = startTime.AddYears(3), Limit = 10 })).FirstOrDefault(); this.UpdateFirstKnownData(metaData.HistoryId, KlineToCandlestick(firstAvailable)); } //if we already downloaded the first available we can start download from the last known //otherwise we leave start time as it is to avoid creating holes if (metaData.FirstKnownData.CloseTime.AddSeconds(1) >= metaData.FirstBar.OpenTime) { startTime = new DateTime(metaData.LastBar.Time.Ticks, DateTimeKind.Utc).Subtract(redownloadStart); } // avoid downloading again some data if possible if (endTime <= metaData.LastBar.Time) { endTime = metaData.FirstBar.Time; } } } //download and add bars if needed if (endTime > startTime) { var candlesDownloaded = await DownloadCandles(symbol, startTime, endTime); //--- this.AddCandlesticks(symId, candlesDownloaded); this.ValidateData(symId); this.SaveAndClose(symId); } Console.WriteLine($"{symbol} history downloaded"); } catch (Exception ex) { var msg = $"Fatal Exception during {symbol} history download: "; if (ex is BinanceException binException) { msg += binException.ErrorDetails; } else { msg += ex.Message; } Console.WriteLine(msg); } finally { semaphore.Release(); } }
public override ISymbolHistory GetSymbolHistory(SymbolHistoryId info, DateTime startOfData, DateTime endOfData) { lock (LastHistoryRequest) LastHistoryRequest[info.Key] = DateTime.Now; return(base.GetSymbolHistory(info, startOfData, endOfData)); }