/// <summary>
/// Loads all files then saves them again ( so format is updates )
/// </summary>
private void Update1()
{
//get a
var files = Directory.GetFiles(DataDir, "*.bin");
IEnumerable <IGrouping <string, string> > filesGrouped;
//if (files.Length > 0)
{
files = files.Concat(Directory.GetFiles(DataDir, "*.bin2")).ToArray();
filesGrouped = files.GroupBy(
f =>
{
var info = HistoryChunkId.Parse(f);
return(info.HistoryId.Key);
}
);
foreach (var group in filesGrouped)
{
var info = HistoryChunkId.Parse(group.First());
var histMetadata = new SymbolHistoryMetaDataInternal(info.HistoryId);
foreach (var fpath in group)
{
var fdata = HistoryChunk.Load(fpath);
histMetadata.AddBars(fdata.Ticks);
if (Path.GetExtension(fpath) == ".bin")
{
File.Delete(fpath);
}
}
histMetadata.Save(DataDir);
}
}
}
/// <summary>
/// Saves all data to disk and updates LoadedFiles list
/// </summary>
/// <param name="dataDir"></param>
public void Save_Protobuf(string dataDir)
{
lock (_Ticks)
{
int i = 0;
while (i < this.Ticks.Count)
{
DateTime startDate = new DateTime(this.Ticks[i].OpenTime.Year, this.Ticks[i].OpenTime.Month, 1, 0, 0, 0, DateTimeKind.Utc);
DateTime endDate = startDate.AddMonths(1);
List <Candlestick> candlesOfMont = new List <Candlestick>();
while (i < this.Ticks.Count && this.Ticks[i].OpenTime < endDate)
{
candlesOfMont.Add(new Candlestick(this.Ticks[i]));
i++;
}
HistoryChunkId newChunkId = new HistoryChunkId(this.Id, startDate);
HistoryChunk sdata = new HistoryChunk()
{
ChunkId = newChunkId,
Ticks = candlesOfMont,
};
using (var fs = File.Open(newChunkId.GetFilePath(dataDir), FileMode.Create))
Serializer.Serialize <HistoryChunk>(fs, sdata);
this.LoadedFiles.Add(newChunkId);
}
}
}
public void LoadHistory(string dataDir, DateTime startOfData, DateTime endOfData)
{
startOfData = new DateTime(startOfData.Year, startOfData.Month, 1, 0, 0, 0, DateTimeKind.Utc);
List <DateRange> missingData = new List <DateRange>();
lock (this.Locker)
{
if (this.View?.Ticks == null)
{
this.View = new HistoryView(this.HistoryId);
}
//check if we already have some records and load them
if (this.View.Ticks.Count < 1)
{
missingData.Add(new DateRange(startOfData, endOfData));
}
else
{
if (this.View.StartOfData > startOfData)
{
missingData.Add(new DateRange(startOfData, this.View.StartOfData));
}
if (endOfData > this.View.EndOfData)
{
missingData.Add(new DateRange(this.View.EndOfData, endOfData));
}
}
//load missing data
foreach (var finfo in this.Chunks)
{
Debug.Assert(HistoryId.Key == finfo.HistoryId.Key, $"Hist id {HistoryId.Key} - finfo {finfo.HistoryId.Key}");
var dateInRange = missingData.Any(dr => finfo.StartDate >= dr.start && finfo.StartDate < dr.end);
if (dateInRange && this.View.LoadedFiles.Add(finfo)) //if is in any range and not already loaded
{
try
{
HistoryChunk fdata = HistoryChunk.Load(finfo.GetFilePath(dataDir));
this.AddBars(fdata.Ticks.Where(tick => tick.Time <= endOfData));
}
catch (Exception ex)
{
Logger.Error($"ERROR loading file {finfo.GetFilePath(dataDir)}: {ex.Message}");
}
}
}
}
}
public static HistoryChunk Load(string filePath)
{
var fileExtension = Path.GetExtension(filePath);
using (var fs = File.Open(filePath, FileMode.Open))
{
if (fileExtension == ".bin")
{
SymbolHistoryFile_Legacy fdata = Serializer.Deserialize <SymbolHistoryFile_Legacy>(fs);
return(new HistoryChunk(fdata));
}
else if (fileExtension == ".bin2")
{
HistoryChunk fdata = Serializer.Deserialize <HistoryChunk>(fs);
return(fdata);
}
else
{
throw new InvalidOperationException("Wrong extension for loading HistoryChunk.");
}
}
}
private void Update2()
{
var data = new Dictionary <string, SymbolHistoryMetaDataInternal>();
var files = Directory.GetFiles(DataDir, "*.bin2");
foreach (var filePath in files)
{
var fileInfo = HistoryChunkId.Parse(filePath);
//-- get metaData --
if (!data.ContainsKey(fileInfo.HistoryId.Key))
{
data.Add(fileInfo.HistoryId.Key, new SymbolHistoryMetaDataInternal(fileInfo.HistoryId));
}
var symData = data[fileInfo.HistoryId.Key];
symData.Chunks.Add(fileInfo);
//update first and last tick time
HistoryChunk fileData = HistoryChunk.Load(filePath);
if (fileData.Ticks.Count > 0)
{
symData.UpdateLastBar(fileData.Ticks.First());
symData.UpdateLastBar(fileData.Ticks.Last());
}
}
//reinsert everything
DbSymbolsMetaData.DeleteAll();
Db.Checkpoint();
var allData = data.Values.ToArray();
foreach (var dat in allData)
{
DbSymbolsMetaData.Upsert(dat);
}
Db.Checkpoint();
}
private void ValidateSymbolsMetadata()
{
foreach (var symData in SymbolsMetaData.Values.ToArray())
{
bool ok = true;
var chunks = symData.Chunks.ToList();
symData.Chunks.Clear();
foreach (var chunk in chunks)
{
bool canAddChunk = true;
var newPath = chunk.GetFilePath(DataDir);
//check that file exists
if (File.Exists(newPath))
{
var fileInfo = new FileInfo(newPath);
//check that file size is plausible
if (fileInfo.Length > 3e6)
{
Logger.Error("Error: history file {0} has anomalous size. It will be deleted", newPath);
ok = false;
File.Delete(newPath);
canAddChunk = false;
}
//check that id is right
if (chunk.HistoryId.Key != symData.HistoryId.Key)
{
Logger.Error("Error: history file {0} has wrong history id {1}", newPath, chunk.HistoryId.Key);
File.Delete(newPath);
canAddChunk = false;
}
}
else
{
canAddChunk = false;
ok = false;
Logger.Error("Error: history file {0} does not exist.", newPath);
}
if (canAddChunk)
{
symData.Chunks.Add(new HistoryChunkId(chunk.HistoryId, chunk.StartDate));
}
}
//rebuild history if needed
if (ok == false)
{
Logger.Info("Rebuilding corrupted {0} history", symData.Id);
symData.ClearView();
var newSymData = new SymbolHistoryMetaDataInternal(symData.HistoryId);
foreach (var chunk in symData.Chunks)
{
var chunkData = HistoryChunk.Load(chunk.GetFilePath(DataDir));
newSymData.AddBars(chunkData.Ticks);
}
//save data
newSymData.Save(this.DataDir);
//update dictionary
SymbolsMetaData[newSymData.Id] = newSymData;
//update db
DbSymbolsMetaData.Upsert(newSymData);
newSymData.Validated = true;
}
symData.Validated = true;
}
}
