private List <dynamic> SetNewDynamicListForKeysValues(SortDefinitions sortDefinitions, SQLiteDataReader sdr) { var dynList = new List <dynamic>(); List <SortDefinition> srtKeys = sortDefinitions.GetKeys(); for (int i = 0; i < sortDefinitions.GetKeys().Count; i++) { if (srtKeys[i].IsUniqueKey) { dynList.Add(sdr["SortKey" + i.ToString()]); } } return(dynList); }
private List <dynamic> GetNewDynamicListForKeys(SortDefinitions sortDefinitions) { var dynList = new List <dynamic>(); List <SortDefinition> srtKeys = sortDefinitions.GetKeys(); for (int i = 0; i < sortDefinitions.GetKeys().Count; i++) { if (srtKeys[i].IsUniqueKey) { dynList.Add(null); } } return(dynList); }
private void InsertBulk() { int numberOfKeys = sortDefs.GetKeys().Count; List <SortDefinition> sortKeys = sortDefs.GetKeys(); using (var cmd = new SQLiteCommand(dbConnection)) { using (var transaction = dbConnection.BeginTransaction()) { cmd.CommandText = insertBulkCmd; for (int pvdx = 0; pvdx < numberOfKeys; pvdx++) { cmd.Parameters.AddWithValue(string.Format("@key{0}", pvdx), string.Format("SortKey{0}", pvdx)); } cmd.Parameters.AddWithValue("@data", "LineData"); foreach (var item in sortKeyDataList) { for (int i = 0; i < numberOfKeys; i++) { if (sortKeys[i].DataType == KeyType.Numberic) { cmd.Parameters[string.Format("@key{0}", i)].Value = Convert.ToInt64(item.KeyValues[i]); } else { cmd.Parameters[string.Format("@key{0}", i)].Value = Convert.ToString(item.KeyValues[i]); } } cmd.Parameters["@data"].Value = item.Data; cmd.ExecuteNonQuery(); } transaction.Commit(); } } sortKeyDataList.Clear(); }
internal static SortResults SortFixedWidthByKeyDefCore(string sourcefilePath, SortDefinitions sortDefinitions, Action<string, string[]> setKeys, Func<string, bool> dataFilter = null, string destinationFolder = null, bool hasHeader = true, bool returnDuplicates = false, Action<SortProgress> progress = null, DataTransportation dataTransportation = null, bool deleteDbConnPath = true, bool writeOutSortFile = true, int maxBatchSize = 250000) { ArgumentValidation.Validate(sourcefilePath, setKeys, destinationFolder); SortVars srtVars = new SortVars(sourcefilePath, destinationFolder); SortResults srtResults = new SortResults(sourcefilePath, srtVars.DestFolder, srtVars.DbConnPath); SortProgress srtProgress = new SortProgress(); try { srtResults.DeleteDuplicatesFile(); int lineCount = 1; using (StreamReader reader = new StreamReader(sourcefilePath)) using (SqliteSortDefBulkInserter sortBulkInserter = new SqliteSortDefBulkInserter(srtVars.DbConnPath, sortDefinitions, maxBatchSize)) { string line; srtVars.Header = GetHeader(hasHeader, reader); srtProgress.InitReading(); while ((line = reader.ReadLine()) != null) { srtResults.IncrementLinesRead(); ReportReadProgress(progress, srtProgress, srtResults.LinesRead); if (dataFilter == null || dataFilter(line)) { string[] keyValues = new string[sortDefinitions.GetKeys().Count]; setKeys(line, keyValues); sortBulkInserter.Add(new SortKeyData { KeyValues = keyValues, Data = (line + Constants.Common.PreserveCharacter).Compress() }); lineCount++; } else { srtResults.IncrementFiltered(); } } sortBulkInserter.InsertAnyLeftOvers(); sortBulkInserter.AddUnUniqueIndex(); } srtProgress.InitWriting(); if (writeOutSortFile) { srtResults.WriteOutSorted(dbConnPath: srtVars.DbConnPath, header: srtVars.Header, sortDefinitions: sortDefinitions, delimiter: Constants.Delimiters.Tab, returnDuplicates: returnDuplicates, dupesFilePath: srtResults.DuplicatesFilePath, compressed: true, progress: (counter) => { srtProgress.Counter = counter; if (progress != null) { progress(srtProgress); } }, dataTransportation: dataTransportation, deleteDb: deleteDbConnPath); } else { srtResults.Header = srtVars.Header; } srtResults.DeleteDuplicatesFileIfNoDuplicates(); } catch (Exception) { CleanUp(srtVars, srtResults); srtProgress = null; throw; } return srtResults; }