private static void CleanUp(SortVars srtVars, SortResults srtResults) { SortFileHelpers.ExceptionCleanUp(srtVars, srtResults); srtVars = null; srtResults = null; }
internal void DeleteActionFiles() { SortFileHelpers.DeleteFileIfExists(AddsFilePath); SortFileHelpers.DeleteFileIfExists(DeletesFilePath); SortFileHelpers.DeleteFileIfExists(UpdatesFilePath); SortFileHelpers.DeleteFileIfExists(IgnoredFilePath); }
private static void ExceptionCleanUp(string dbConnPath, MergePurgeResults mgPurgeResults) { SortFileHelpers.DeleteFileIfExists(dbConnPath); SortFileHelpers.DeleteFileIfExists(Path.Combine(Path.GetDirectoryName(dbConnPath), SortFileHelpers.GetDbJournalName(dbConnPath))); mgPurgeResults.DeleteActionFiles(); SortFileHelpers.DeleteFileIfExists(mgPurgeResults.NewMasterFilePath); }
public SortResults(string sourceFilePath, string destFolder, string dbConnPath) { string sortedFileName = SortFileHelpers.GetSortedFileName(sourceFilePath); DuplicatesFilePath = Path.Combine(destFolder, SortFileHelpers.GetDupesFileName(sourceFilePath)); SortedFilePath = Path.Combine(destFolder, sortedFileName); DbConnPath = dbConnPath; }
public SortVars(string sourceFilePath, string destFolder) { DbConnStr = SortFileHelpers.GetRandomDbConnStr(); DbJrnFileName = SortFileHelpers.GetDbJournalName(DbConnStr); SortedFileName = SortFileHelpers.GetSortedFileName(sourceFilePath); SourceDirName = SortFileHelpers.GetSourceDirName(sourceFilePath); DestFolder = SortFileHelpers.GetDestinationFolder(SourceDirName, destFolder); DbConnPath = Path.Combine(DestFolder, DbConnStr); }
private static string[] GetMasterFields <T>(MasterFixedWidthFileSource <T> master, string data) { string outLine = SortFileHelpers.UnEscapeByDelimiter(data.Decompress(), Constants.Delimiters.Tab); string[] results = null; FileParser.ParseFixedWidthString(new StringReader(outLine), (fields, lNum) => { results = fields; }, master.FixedWidths); return(results); }
private static string[] GetMasterFields <T>(MasterDelimitedFileSource <T> master, string data) { string outLine = SortFileHelpers.UnEscapeByDelimiter(data, master.Delimiter); string[] results = null; FileParser.ParseDelimitedString(new StringReader(outLine), (fields, lNum) => { results = fields; }, master.Delimiter); return(results); }
internal void ClearSubFilesIfNoCount() { if (AddCount == 0) { SortFileHelpers.DeleteFileIfExists(AddsFilePath); AddsFilePath = string.Empty; } if (DeleteCount == 0) { SortFileHelpers.DeleteFileIfExists(DeletesFilePath); DeletesFilePath = string.Empty; } if (UpdateCount == 0) { SortFileHelpers.DeleteFileIfExists(UpdatesFilePath); UpdatesFilePath = string.Empty; } if (IgnoreCount == 0) { SortFileHelpers.DeleteFileIfExists(IgnoredFilePath); IgnoredFilePath = string.Empty; } }
internal static SortResults SortDelimitedByKeyDefCore( string sourcefilePath, SortDefinitions sortDefinitions, Action<string[], string, string[]> setKeys, Func<string[], string, bool> dataFilter = null, string destinationFolder = null, string delimiter = Constants.Delimiters.Comma, bool hasHeader = true, bool returnDuplicates = false, Action<SortProgress> progress = null, DataTransportation dataTransportation = null, bool deleteDbConnPath = true, bool writeOutSortFile = true, int maxBatchSize = 250000) { ArgumentValidation.Validate(sourcefilePath, setKeys, delimiter, destinationFolder); SortVars srtVars = new SortVars(sourcefilePath, destinationFolder); SortResults srtResults = new SortResults(sourcefilePath, srtVars.DestFolder, srtVars.DbConnPath); SortProgress srtProgress = new SortProgress(); try { srtResults.DeleteDuplicatesFile(); int lineCount = 1; using (StreamReader reader = new StreamReader(sourcefilePath)) using (SqliteSortDefBulkInserter sortBulkInserter = new SqliteSortDefBulkInserter(srtVars.DbConnPath, sortDefinitions, maxBatchSize)) { string line; srtVars.Header = GetHeader(hasHeader, reader); srtProgress.InitReading(); while ((line = reader.ReadLine()) != null) { srtResults.IncrementLinesRead(); ReportReadProgress(progress, srtProgress, srtResults.LinesRead); FileParser.ParseDelimitedString(new StringReader(line), (fields, lNum) => { if (dataFilter == null || dataFilter(fields, line)) { string[] keyValues = new string[sortDefinitions.GetKeys().Count]; setKeys(fields, line, keyValues); sortBulkInserter.Add(new SortKeyData { KeyValues = keyValues, Data = line + SortFileHelpers.EscapeByDelimiter(delimiter) }); lineCount++; } else { srtResults.IncrementFiltered(); } }, delimiter); } sortBulkInserter.InsertAnyLeftOvers(); sortBulkInserter.AddUnUniqueIndex(); } srtProgress.InitWriting(); if (writeOutSortFile) { srtResults.WriteOutSorted(srtVars.DbConnPath, srtVars.Header, sortDefinitions, delimiter, returnDuplicates: returnDuplicates, dupesFilePath: srtResults.DuplicatesFilePath, progress: (counter) => { srtProgress.Counter = counter; if (progress != null) { progress(srtProgress); } }, dataTransportation: dataTransportation, deleteDb: deleteDbConnPath); } else { srtResults.Header = srtVars.Header; } srtResults.DeleteDuplicatesFileIfNoDuplicates(); } catch (Exception) { CleanUp(srtVars, srtResults); srtProgress = null; throw; } return srtResults; }
private static string GetNewMasterDelimitedData <T>(MasterDelimitedFileSource <T> master, string masterLine) { return(masterLine + SortFileHelpers.EscapeByDelimiter(master.Delimiter)); }
internal void WriteOutSorted(string dbConnPath, string header, SortDirection sortDir, string delimiter = Constants.Delimiters.Comma, bool hasUniqueIndex = false, bool returnDuplicates = false, string dupesFilePath = "", bool compressed = false, Action <int> progress = null, DataTransportation dataTransportation = null, bool deleteDb = true) { bool writeSortedFile = WriteoutSortedFile(dataTransportation); if (writeSortedFile) { DeleteSortedFile(); } this.Header = header; StreamWriter dupeWriter = !string.IsNullOrEmpty(dupesFilePath) ? new StreamWriter(dupesFilePath) : null; StreamWriter sw = writeSortedFile ? new StreamWriter(SortedFilePath) : null; using (sw) using (dupeWriter) { if (!string.IsNullOrWhiteSpace(header)) { if (sw != null) { sw.WriteLine(header); } if (returnDuplicates) { WriteHeaderForDuplicatesFile(true, header, dupeWriter); } } using (var cn = new SQLiteConnection(@"Data Source=" + dbConnPath)) { string selectCmd = "SELECT * FROM FileData ORDER BY SortKey"; if (sortDir == SortDirection.Descending) { selectCmd += " DESC"; } cn.Open(); using (var cmd = new SQLiteCommand(selectCmd, cn)) using (SQLiteDataReader rdr = cmd.ExecuteReader()) { dynamic lastReadKey = null; while (rdr.Read()) { string sqlLiteData = (string)rdr["LineData"]; string sqlLiteoutLine = SortFileHelpers.UnEscapeByDelimiter(compressed ? sqlLiteData.Decompress() : sqlLiteData, delimiter); if (hasUniqueIndex) { dynamic sqlLiteKey = rdr["SortKey"]; if (sqlLiteKey.Equals(lastReadKey)) { if (returnDuplicates) { dupeWriter.WriteLine(sqlLiteoutLine); this.IncrementDuplicates(); } continue; } lastReadKey = sqlLiteKey; } if (sw != null) { sw.WriteLine(sqlLiteoutLine); } IncrementLinesSorted(); ReportProgress(progress, LinesSorted); DoDataTransportPassthrough(dataTransportation, sqlLiteoutLine); } } cn.Close(); } } if (deleteDb) { SortFileHelpers.DeleteFileIfExists(dbConnPath); } }
internal void DeleteSortedFile() { SortFileHelpers.DeleteFileIfExists(SortedFilePath); }
internal void DeleteDuplicatesFile() { SortFileHelpers.DeleteFileIfExists(DuplicatesFilePath); }
internal void WriteOutSorted(string dbConnPath, string header, SortDefinitions sortDefinitions, string delimiter = Constants.Delimiters.Comma, bool returnDuplicates = false, string dupesFilePath = "", bool compressed = false, Action <int> progress = null, DataTransportation dataTransportation = null, bool deleteDb = true) { bool writeSortedFile = WriteoutSortedFile(dataTransportation); if (writeSortedFile) { DeleteSortedFile(); } this.Header = header; StreamWriter dupeWriter = !string.IsNullOrEmpty(dupesFilePath) ? new StreamWriter(dupesFilePath) : null; StreamWriter sw = writeSortedFile ? new StreamWriter(SortedFilePath) : null; using (sw) using (dupeWriter) { if (!string.IsNullOrWhiteSpace(header)) { if (sw != null) { sw.WriteLine(header); } if (returnDuplicates) { WriteHeaderForDuplicatesFile(true, header, dupeWriter); } } using (var cn = new SQLiteConnection(@"Data Source=" + dbConnPath)) { string selectCmd = "SELECT * FROM FileData ORDER BY " + sortDefinitions.BuildOrderClause(); cn.Open(); using (var cmd = new SQLiteCommand(selectCmd, cn)) using (SQLiteDataReader rdr = cmd.ExecuteReader()) { var lastReadKeyList = GetNewDynamicListForKeys(sortDefinitions); while (rdr.Read()) { string sqlLiteData = (string)rdr["LineData"]; string sqlLiteoutLine = SortFileHelpers.UnEscapeByDelimiter(compressed ? sqlLiteData.Decompress() : sqlLiteData, delimiter); if (lastReadKeyList.Count > 0) { var currentReadKeyList = SetNewDynamicListForKeysValues(sortDefinitions, rdr); if (KeysEqual(currentReadKeyList, lastReadKeyList)) { if (returnDuplicates) { dupeWriter.WriteLine(sqlLiteoutLine); this.IncrementDuplicates(); } continue; } lastReadKeyList = currentReadKeyList; } if (sw != null) { sw.WriteLine(sqlLiteoutLine); } IncrementLinesSorted(); ReportProgress(progress, LinesSorted); DoDataTransportPassthrough(dataTransportation, sqlLiteoutLine); } } cn.Close(); } } if (deleteDb) { SortFileHelpers.DeleteFileIfExists(dbConnPath); } }