/// <summary> /// Loads all files then saves them again ( so format is updates ) /// </summary> private void Update1() { //get a var files = Directory.GetFiles(DataDir, "*.bin"); IEnumerable <IGrouping <string, string> > filesGrouped; //if (files.Length > 0) { files = files.Concat(Directory.GetFiles(DataDir, "*.bin2")).ToArray(); filesGrouped = files.GroupBy( f => { var info = HistoryChunkId.Parse(f); return(info.HistoryId.Key); } ); foreach (var group in filesGrouped) { var info = HistoryChunkId.Parse(group.First()); var histMetadata = new SymbolHistoryMetaDataInternal(info.HistoryId); foreach (var fpath in group) { var fdata = HistoryChunk.Load(fpath); histMetadata.AddBars(fdata.Ticks); if (Path.GetExtension(fpath) == ".bin") { File.Delete(fpath); } } histMetadata.Save(DataDir); } } }
public void LoadHistory(string dataDir, DateTime startOfData, DateTime endOfData) { startOfData = new DateTime(startOfData.Year, startOfData.Month, 1, 0, 0, 0, DateTimeKind.Utc); List <DateRange> missingData = new List <DateRange>(); lock (this.Locker) { if (this.View?.Ticks == null) { this.View = new HistoryView(this.HistoryId); } //check if we already have some records and load them if (this.View.Ticks.Count < 1) { missingData.Add(new DateRange(startOfData, endOfData)); } else { if (this.View.StartOfData > startOfData) { missingData.Add(new DateRange(startOfData, this.View.StartOfData)); } if (endOfData > this.View.EndOfData) { missingData.Add(new DateRange(this.View.EndOfData, endOfData)); } } //load missing data foreach (var finfo in this.Chunks) { Debug.Assert(HistoryId.Key == finfo.HistoryId.Key, $"Hist id {HistoryId.Key} - finfo {finfo.HistoryId.Key}"); var dateInRange = missingData.Any(dr => finfo.StartDate >= dr.start && finfo.StartDate < dr.end); if (dateInRange && this.View.LoadedFiles.Add(finfo)) //if is in any range and not already loaded { try { HistoryChunk fdata = HistoryChunk.Load(finfo.GetFilePath(dataDir)); this.AddBars(fdata.Ticks.Where(tick => tick.Time <= endOfData)); } catch (Exception ex) { Logger.Error($"ERROR loading file {finfo.GetFilePath(dataDir)}: {ex.Message}"); } } } } }
private void Update2() { var data = new Dictionary <string, SymbolHistoryMetaDataInternal>(); var files = Directory.GetFiles(DataDir, "*.bin2"); foreach (var filePath in files) { var fileInfo = HistoryChunkId.Parse(filePath); //-- get metaData -- if (!data.ContainsKey(fileInfo.HistoryId.Key)) { data.Add(fileInfo.HistoryId.Key, new SymbolHistoryMetaDataInternal(fileInfo.HistoryId)); } var symData = data[fileInfo.HistoryId.Key]; symData.Chunks.Add(fileInfo); //update first and last tick time HistoryChunk fileData = HistoryChunk.Load(filePath); if (fileData.Ticks.Count > 0) { symData.UpdateLastBar(fileData.Ticks.First()); symData.UpdateLastBar(fileData.Ticks.Last()); } } //reinsert everything DbSymbolsMetaData.DeleteAll(); Db.Checkpoint(); var allData = data.Values.ToArray(); foreach (var dat in allData) { DbSymbolsMetaData.Upsert(dat); } Db.Checkpoint(); }
private void ValidateSymbolsMetadata() { foreach (var symData in SymbolsMetaData.Values.ToArray()) { bool ok = true; var chunks = symData.Chunks.ToList(); symData.Chunks.Clear(); foreach (var chunk in chunks) { bool canAddChunk = true; var newPath = chunk.GetFilePath(DataDir); //check that file exists if (File.Exists(newPath)) { var fileInfo = new FileInfo(newPath); //check that file size is plausible if (fileInfo.Length > 3e6) { Logger.Error("Error: history file {0} has anomalous size. It will be deleted", newPath); ok = false; File.Delete(newPath); canAddChunk = false; } //check that id is right if (chunk.HistoryId.Key != symData.HistoryId.Key) { Logger.Error("Error: history file {0} has wrong history id {1}", newPath, chunk.HistoryId.Key); File.Delete(newPath); canAddChunk = false; } } else { canAddChunk = false; ok = false; Logger.Error("Error: history file {0} does not exist.", newPath); } if (canAddChunk) { symData.Chunks.Add(new HistoryChunkId(chunk.HistoryId, chunk.StartDate)); } } //rebuild history if needed if (ok == false) { Logger.Info("Rebuilding corrupted {0} history", symData.Id); symData.ClearView(); var newSymData = new SymbolHistoryMetaDataInternal(symData.HistoryId); foreach (var chunk in symData.Chunks) { var chunkData = HistoryChunk.Load(chunk.GetFilePath(DataDir)); newSymData.AddBars(chunkData.Ticks); } //save data newSymData.Save(this.DataDir); //update dictionary SymbolsMetaData[newSymData.Id] = newSymData; //update db DbSymbolsMetaData.Upsert(newSymData); newSymData.Validated = true; } symData.Validated = true; } }