/// <summary> /// Loads all files then saves them again ( so format is updates ) /// </summary> private void Update1() { //get a var files = Directory.GetFiles(DataDir, "*.bin"); IEnumerable <IGrouping <string, string> > filesGrouped; //if (files.Length > 0) { files = files.Concat(Directory.GetFiles(DataDir, "*.bin2")).ToArray(); filesGrouped = files.GroupBy( f => { var info = HistoryChunkId.Parse(f); return(info.HistoryId.Key); } ); foreach (var group in filesGrouped) { var info = HistoryChunkId.Parse(group.First()); var histMetadata = new SymbolHistoryMetaDataInternal(info.HistoryId); foreach (var fpath in group) { var fdata = HistoryChunk.Load(fpath); histMetadata.AddBars(fdata.Ticks); if (Path.GetExtension(fpath) == ".bin") { File.Delete(fpath); } } histMetadata.Save(DataDir); } } }
/// <summary> /// Saves all data to disk and updates LoadedFiles list /// </summary> /// <param name="dataDir"></param> public void Save_Protobuf(string dataDir) { lock (_Ticks) { int i = 0; while (i < this.Ticks.Count) { DateTime startDate = new DateTime(this.Ticks[i].OpenTime.Year, this.Ticks[i].OpenTime.Month, 1, 0, 0, 0, DateTimeKind.Utc); DateTime endDate = startDate.AddMonths(1); List <Candlestick> candlesOfMont = new List <Candlestick>(); while (i < this.Ticks.Count && this.Ticks[i].OpenTime < endDate) { candlesOfMont.Add(new Candlestick(this.Ticks[i])); i++; } HistoryChunkId newChunkId = new HistoryChunkId(this.Id, startDate); HistoryChunk sdata = new HistoryChunk() { ChunkId = newChunkId, Ticks = candlesOfMont, }; using (var fs = File.Open(newChunkId.GetFilePath(dataDir), FileMode.Create)) Serializer.Serialize <HistoryChunk>(fs, sdata); this.LoadedFiles.Add(newChunkId); } } }
public void LoadHistory(string dataDir, DateTime startOfData, DateTime endOfData) { startOfData = new DateTime(startOfData.Year, startOfData.Month, 1, 0, 0, 0, DateTimeKind.Utc); List <DateRange> missingData = new List <DateRange>(); lock (this.Locker) { if (this.View?.Ticks == null) { this.View = new HistoryView(this.HistoryId); } //check if we already have some records and load them if (this.View.Ticks.Count < 1) { missingData.Add(new DateRange(startOfData, endOfData)); } else { if (this.View.StartOfData > startOfData) { missingData.Add(new DateRange(startOfData, this.View.StartOfData)); } if (endOfData > this.View.EndOfData) { missingData.Add(new DateRange(this.View.EndOfData, endOfData)); } } //load missing data foreach (var finfo in this.Chunks) { Debug.Assert(HistoryId.Key == finfo.HistoryId.Key, $"Hist id {HistoryId.Key} - finfo {finfo.HistoryId.Key}"); var dateInRange = missingData.Any(dr => finfo.StartDate >= dr.start && finfo.StartDate < dr.end); if (dateInRange && this.View.LoadedFiles.Add(finfo)) //if is in any range and not already loaded { try { HistoryChunk fdata = HistoryChunk.Load(finfo.GetFilePath(dataDir)); this.AddBars(fdata.Ticks.Where(tick => tick.Time <= endOfData)); } catch (Exception ex) { Logger.Error($"ERROR loading file {finfo.GetFilePath(dataDir)}: {ex.Message}"); } } } } }
public static HistoryChunk Load(string filePath) { var fileExtension = Path.GetExtension(filePath); using (var fs = File.Open(filePath, FileMode.Open)) { if (fileExtension == ".bin") { SymbolHistoryFile_Legacy fdata = Serializer.Deserialize <SymbolHistoryFile_Legacy>(fs); return(new HistoryChunk(fdata)); } else if (fileExtension == ".bin2") { HistoryChunk fdata = Serializer.Deserialize <HistoryChunk>(fs); return(fdata); } else { throw new InvalidOperationException("Wrong extension for loading HistoryChunk."); } } }
private void Update2() { var data = new Dictionary <string, SymbolHistoryMetaDataInternal>(); var files = Directory.GetFiles(DataDir, "*.bin2"); foreach (var filePath in files) { var fileInfo = HistoryChunkId.Parse(filePath); //-- get metaData -- if (!data.ContainsKey(fileInfo.HistoryId.Key)) { data.Add(fileInfo.HistoryId.Key, new SymbolHistoryMetaDataInternal(fileInfo.HistoryId)); } var symData = data[fileInfo.HistoryId.Key]; symData.Chunks.Add(fileInfo); //update first and last tick time HistoryChunk fileData = HistoryChunk.Load(filePath); if (fileData.Ticks.Count > 0) { symData.UpdateLastBar(fileData.Ticks.First()); symData.UpdateLastBar(fileData.Ticks.Last()); } } //reinsert everything DbSymbolsMetaData.DeleteAll(); Db.Checkpoint(); var allData = data.Values.ToArray(); foreach (var dat in allData) { DbSymbolsMetaData.Upsert(dat); } Db.Checkpoint(); }
private void ValidateSymbolsMetadata() { foreach (var symData in SymbolsMetaData.Values.ToArray()) { bool ok = true; var chunks = symData.Chunks.ToList(); symData.Chunks.Clear(); foreach (var chunk in chunks) { bool canAddChunk = true; var newPath = chunk.GetFilePath(DataDir); //check that file exists if (File.Exists(newPath)) { var fileInfo = new FileInfo(newPath); //check that file size is plausible if (fileInfo.Length > 3e6) { Logger.Error("Error: history file {0} has anomalous size. It will be deleted", newPath); ok = false; File.Delete(newPath); canAddChunk = false; } //check that id is right if (chunk.HistoryId.Key != symData.HistoryId.Key) { Logger.Error("Error: history file {0} has wrong history id {1}", newPath, chunk.HistoryId.Key); File.Delete(newPath); canAddChunk = false; } } else { canAddChunk = false; ok = false; Logger.Error("Error: history file {0} does not exist.", newPath); } if (canAddChunk) { symData.Chunks.Add(new HistoryChunkId(chunk.HistoryId, chunk.StartDate)); } } //rebuild history if needed if (ok == false) { Logger.Info("Rebuilding corrupted {0} history", symData.Id); symData.ClearView(); var newSymData = new SymbolHistoryMetaDataInternal(symData.HistoryId); foreach (var chunk in symData.Chunks) { var chunkData = HistoryChunk.Load(chunk.GetFilePath(DataDir)); newSymData.AddBars(chunkData.Ticks); } //save data newSymData.Save(this.DataDir); //update dictionary SymbolsMetaData[newSymData.Id] = newSymData; //update db DbSymbolsMetaData.Upsert(newSymData); newSymData.Validated = true; } symData.Validated = true; } }