public virtual ISymbolHistory GetSymbolHistory(SymbolHistoryId info, DateTime startOfData, DateTime endOfData)
{
var meta = GetMetaDataInternal(info);
meta.LoadHistory(DataDir, startOfData, endOfData);
return(new SymbolHistory(meta.View, startOfData, endOfData));
}
public SymbolHistoryMetaDataInternal(SymbolHistoryId historyId)
{
HistoryId = historyId;
Id = historyId.Key;
Chunks = new HashSet <HistoryChunkId>();
GapsConfirmed = new List <DateRange>();
GapsUnconfirmed = new List <DateRange>();
View = new HistoryView(historyId);
}
private SymbolHistoryMetaDataInternal GetMetaDataInternal(SymbolHistoryId historyInfo)
{
lock (SymbolsMetaData)
{
if (!SymbolsMetaData.TryGetValue(historyInfo.Key, out var metaData))
{
metaData = new SymbolHistoryMetaDataInternal(historyInfo);
SymbolsMetaData.Add(historyInfo.Key, metaData);
}
return(metaData);
}
}
public void AddCandlesticks(SymbolHistoryId symId, IEnumerable <Candlestick> candles)
{
//load all available data in date rage
if (candles.Any())
{
var meta = GetMetaDataInternal(symId);
meta.LoadHistory(DataDir, candles.First().OpenTime, DateTime.MaxValue);
//add data
//Debug.Assert(sdata.Ticks.Count > 0);
//Debug.Assert(candles.First().OpenTime > sdata.Ticks.First().OpenTime, "Error in sdata times");
meta.AddBars(candles);
}
}
public void SaveAndClose(SymbolHistoryId info, bool save = true)
{
var meta = GetMetaDataInternal(info);
if (meta == null)
{
throw new Exception("symbol history not found");
}
if (save)
{
Save(meta);
}
meta.ClearView();
}
public void ValidateData(SymbolHistoryId finfo)
{
lock (SymbolsMetaData)
{
var myData = GetMetaDataInternal(finfo);
if (myData.View != null)
{
var ticks = myData.View.Ticks;
for (int i = 1; i < ticks.Count; i++)
{
if (ticks[i].OpenTime < ticks[i - 1].OpenTime)
{
Console.WriteLine($"{finfo.Market} - {finfo.Symbol} - {finfo.Resolution} -> bad data at {i}");
}
}
}
}
}
public void Delete(string market, string symbol, TimeSpan time)
{
var histInfo = new SymbolHistoryId(market, symbol, time);
lock (SymbolsMetaData)
{
var meta = GetMetaDataInternal(histInfo);
if (meta == null)
{
throw new Exception("symbol history not found");
}
foreach (var fileInfo in meta.Chunks.ToArray())
{
if (File.Exists(fileInfo.GetFilePath(DataDir)))
{
File.Delete(fileInfo.GetFilePath(DataDir));
}
meta.Chunks.Remove(fileInfo);
}
}
}
internal bool Equals(SymbolHistoryId info)
{
return(info.Market == Market && info.Symbol == Symbol && info.Resolution == Timeframe);
}
internal void UpdateFirstKnownData(SymbolHistoryId info, ITradeBar firstAvailable)
{
var data = GetMetaDataInternal(info);
data.FirstKnownData = firstAvailable;
}
public ISymbolHistory GetSymbolHistory(SymbolHistoryId info)
{
return(GetSymbolHistory(info, DateTime.MinValue));
}
public ISymbolHistory GetSymbolHistory(SymbolHistoryId info, DateTime startOfData)
{
return(GetSymbolHistory(info, startOfData, DateTime.MaxValue));
}
public SymbolHistoryMetaData GetMetaData(SymbolHistoryId historyInfo) => GetMetaDataInternal(historyInfo);
public void DownloadMissingData(SymbolHistoryId histInfo, DateTime fromTime, DateTime toTime, Func <string, DateTime, DateTime, Candlestick[]> downloadCandlesCallback)
{
int duplicates = 0;
int matchErrors = 0;
Console.WriteLine($"Checking {histInfo.Symbol} history data ");
var meta = GetMetaDataInternal(histInfo);
meta.LoadHistory(DataDir, fromTime, toTime);
var oldTicks = meta.View.Ticks;
var newTicks = new List <Candlestick>();
Candlestick lastAdded() => newTicks[newTicks.Count - 1];
if (oldTicks.Count > 0)
{
newTicks.Add(oldTicks[0]);
for (int i = 1; i < oldTicks.Count; i++)
{
var candleToAdd = oldTicks[i];
if (candleToAdd.OpenTime == lastAdded().OpenTime)
{
//we skip this
if (candleToAdd.Equals(lastAdded()))
{
matchErrors++;
}
duplicates++;
}
else if (candleToAdd.OpenTime < lastAdded().OpenTime)
{
Console.WriteLine($" Bad candle at {candleToAdd.OpenTime}");
}
else
{
//check holes
if (candleToAdd.OpenTime - lastAdded().OpenTime > histInfo.Resolution)
{
//hole
Console.WriteLine($" Hole found {lastAdded().OpenTime} -> {candleToAdd.OpenTime}");
//redownload data
if (downloadCandlesCallback != null)
{
var candlesToAdd = downloadCandlesCallback(histInfo.Symbol, lastAdded().OpenTime, candleToAdd.OpenTime)
.Where(c => c.OpenTime > lastAdded().OpenTime)
.OrderBy(c => c.OpenTime).ToArray();
newTicks.AddRange(candlesToAdd);
Console.WriteLine($" Hole fixed!{candlesToAdd.FirstOrDefault()?.OpenTime} -> {candlesToAdd.LastOrDefault()?.OpenTime} ");
}
}
//finally add the candle
if (candleToAdd.OpenTime > lastAdded().OpenTime)
{
newTicks.Add(candleToAdd);
}
}
}
meta.View.Ticks.Clear();
meta.View.Ticks.AddRange(newTicks);
SaveAndClose(histInfo);
}
Console.WriteLine($" Fixing {histInfo.Symbol} completed: duplicates {duplicates} - matchErrors {matchErrors}");
}
public HistoryView(SymbolHistoryId id)
{
Id = id;
_Ticks = new List <Candlestick>();
}