/// <summary> /// Loads all files then saves them again ( so format is updates ) /// </summary> private void Update1() { //get a var files = Directory.GetFiles(DataDir, "*.bin"); IEnumerable <IGrouping <string, string> > filesGrouped; //if (files.Length > 0) { files = files.Concat(Directory.GetFiles(DataDir, "*.bin2")).ToArray(); filesGrouped = files.GroupBy( f => { var info = HistoryChunkId.Parse(f); return(info.HistoryId.Key); } ); foreach (var group in filesGrouped) { var info = HistoryChunkId.Parse(group.First()); var histMetadata = new SymbolHistoryMetaDataInternal(info.HistoryId); foreach (var fpath in group) { var fdata = HistoryChunk.Load(fpath); histMetadata.AddBars(fdata.Ticks); if (Path.GetExtension(fpath) == ".bin") { File.Delete(fpath); } } histMetadata.Save(DataDir); } } }
private void Save(SymbolHistoryMetaDataInternal sdata) { lock (DbSymbolsMetaData) { sdata.Save(DataDir); DbSymbolsMetaData.Upsert(sdata); } }
private SymbolHistoryMetaDataInternal GetMetaDataInternal(SymbolHistoryId historyInfo) { lock (SymbolsMetaData) { if (!SymbolsMetaData.TryGetValue(historyInfo.Key, out var metaData)) { metaData = new SymbolHistoryMetaDataInternal(historyInfo); SymbolsMetaData.Add(historyInfo.Key, metaData); } return(metaData); } }
private void ValidateSymbolsMetadata() { foreach (var symData in SymbolsMetaData.Values.ToArray()) { bool ok = true; var chunks = symData.Chunks.ToList(); symData.Chunks.Clear(); foreach (var chunk in chunks) { bool canAddChunk = true; var newPath = chunk.GetFilePath(DataDir); //check that file exists if (File.Exists(newPath)) { var fileInfo = new FileInfo(newPath); //check that file size is plausible if (fileInfo.Length > 3e6) { Logger.Error("Error: history file {0} has anomalous size. It will be deleted", newPath); ok = false; File.Delete(newPath); canAddChunk = false; } //check that id is right if (chunk.HistoryId.Key != symData.HistoryId.Key) { Logger.Error("Error: history file {0} has wrong history id {1}", newPath, chunk.HistoryId.Key); File.Delete(newPath); canAddChunk = false; } } else { canAddChunk = false; ok = false; Logger.Error("Error: history file {0} does not exist.", newPath); } if (canAddChunk) { symData.Chunks.Add(new HistoryChunkId(chunk.HistoryId, chunk.StartDate)); } } //rebuild history if needed if (ok == false) { Logger.Info("Rebuilding corrupted {0} history", symData.Id); symData.ClearView(); var newSymData = new SymbolHistoryMetaDataInternal(symData.HistoryId); foreach (var chunk in symData.Chunks) { var chunkData = HistoryChunk.Load(chunk.GetFilePath(DataDir)); newSymData.AddBars(chunkData.Ticks); } //save data newSymData.Save(this.DataDir); //update dictionary SymbolsMetaData[newSymData.Id] = newSymData; //update db DbSymbolsMetaData.Upsert(newSymData); newSymData.Validated = true; } symData.Validated = true; } }