/// <summary>
/// Loads all files then saves them again ( so format is updates )
/// </summary>
private void Update1()
{
//get a
var files = Directory.GetFiles(DataDir, "*.bin");
IEnumerable <IGrouping <string, string> > filesGrouped;
//if (files.Length > 0)
{
files = files.Concat(Directory.GetFiles(DataDir, "*.bin2")).ToArray();
filesGrouped = files.GroupBy(
f =>
{
var info = HistoryChunkId.Parse(f);
return(info.HistoryId.Key);
}
);
foreach (var group in filesGrouped)
{
var info = HistoryChunkId.Parse(group.First());
var histMetadata = new SymbolHistoryMetaDataInternal(info.HistoryId);
foreach (var fpath in group)
{
var fdata = HistoryChunk.Load(fpath);
histMetadata.AddBars(fdata.Ticks);
if (Path.GetExtension(fpath) == ".bin")
{
File.Delete(fpath);
}
}
histMetadata.Save(DataDir);
}
}
}
private void Save(SymbolHistoryMetaDataInternal sdata)
{
lock (DbSymbolsMetaData)
{
sdata.Save(DataDir);
DbSymbolsMetaData.Upsert(sdata);
}
}
private SymbolHistoryMetaDataInternal GetMetaDataInternal(SymbolHistoryId historyInfo)
{
lock (SymbolsMetaData)
{
if (!SymbolsMetaData.TryGetValue(historyInfo.Key, out var metaData))
{
metaData = new SymbolHistoryMetaDataInternal(historyInfo);
SymbolsMetaData.Add(historyInfo.Key, metaData);
}
return(metaData);
}
}
private void ValidateSymbolsMetadata()
{
foreach (var symData in SymbolsMetaData.Values.ToArray())
{
bool ok = true;
var chunks = symData.Chunks.ToList();
symData.Chunks.Clear();
foreach (var chunk in chunks)
{
bool canAddChunk = true;
var newPath = chunk.GetFilePath(DataDir);
//check that file exists
if (File.Exists(newPath))
{
var fileInfo = new FileInfo(newPath);
//check that file size is plausible
if (fileInfo.Length > 3e6)
{
Logger.Error("Error: history file {0} has anomalous size. It will be deleted", newPath);
ok = false;
File.Delete(newPath);
canAddChunk = false;
}
//check that id is right
if (chunk.HistoryId.Key != symData.HistoryId.Key)
{
Logger.Error("Error: history file {0} has wrong history id {1}", newPath, chunk.HistoryId.Key);
File.Delete(newPath);
canAddChunk = false;
}
}
else
{
canAddChunk = false;
ok = false;
Logger.Error("Error: history file {0} does not exist.", newPath);
}
if (canAddChunk)
{
symData.Chunks.Add(new HistoryChunkId(chunk.HistoryId, chunk.StartDate));
}
}
//rebuild history if needed
if (ok == false)
{
Logger.Info("Rebuilding corrupted {0} history", symData.Id);
symData.ClearView();
var newSymData = new SymbolHistoryMetaDataInternal(symData.HistoryId);
foreach (var chunk in symData.Chunks)
{
var chunkData = HistoryChunk.Load(chunk.GetFilePath(DataDir));
newSymData.AddBars(chunkData.Ticks);
}
//save data
newSymData.Save(this.DataDir);
//update dictionary
SymbolsMetaData[newSymData.Id] = newSymData;
//update db
DbSymbolsMetaData.Upsert(newSymData);
newSymData.Validated = true;
}
symData.Validated = true;
}
}