private List <dynamic> SetNewDynamicListForKeysValues(SortDefinitions sortDefinitions, SQLiteDataReader sdr)
{
var dynList = new List <dynamic>();
List <SortDefinition> srtKeys = sortDefinitions.GetKeys();
for (int i = 0; i < sortDefinitions.GetKeys().Count; i++)
{
if (srtKeys[i].IsUniqueKey)
{
dynList.Add(sdr["SortKey" + i.ToString()]);
}
}
return(dynList);
}
private List <dynamic> GetNewDynamicListForKeys(SortDefinitions sortDefinitions)
{
var dynList = new List <dynamic>();
List <SortDefinition> srtKeys = sortDefinitions.GetKeys();
for (int i = 0; i < sortDefinitions.GetKeys().Count; i++)
{
if (srtKeys[i].IsUniqueKey)
{
dynList.Add(null);
}
}
return(dynList);
}
private void InsertBulk()
{
int numberOfKeys = sortDefs.GetKeys().Count;
List <SortDefinition> sortKeys = sortDefs.GetKeys();
using (var cmd = new SQLiteCommand(dbConnection))
{
using (var transaction = dbConnection.BeginTransaction())
{
cmd.CommandText = insertBulkCmd;
for (int pvdx = 0; pvdx < numberOfKeys; pvdx++)
{
cmd.Parameters.AddWithValue(string.Format("@key{0}", pvdx), string.Format("SortKey{0}", pvdx));
}
cmd.Parameters.AddWithValue("@data", "LineData");
foreach (var item in sortKeyDataList)
{
for (int i = 0; i < numberOfKeys; i++)
{
if (sortKeys[i].DataType == KeyType.Numberic)
{
cmd.Parameters[string.Format("@key{0}", i)].Value = Convert.ToInt64(item.KeyValues[i]);
}
else
{
cmd.Parameters[string.Format("@key{0}", i)].Value = Convert.ToString(item.KeyValues[i]);
}
}
cmd.Parameters["@data"].Value = item.Data;
cmd.ExecuteNonQuery();
}
transaction.Commit();
}
}
sortKeyDataList.Clear();
}
internal static SortResults SortFixedWidthByKeyDefCore(string sourcefilePath,
SortDefinitions sortDefinitions,
Action<string, string[]> setKeys,
Func<string, bool> dataFilter = null,
string destinationFolder = null,
bool hasHeader = true,
bool returnDuplicates = false,
Action<SortProgress> progress = null,
DataTransportation dataTransportation = null,
bool deleteDbConnPath = true,
bool writeOutSortFile = true,
int maxBatchSize = 250000)
{
ArgumentValidation.Validate(sourcefilePath, setKeys, destinationFolder);
SortVars srtVars = new SortVars(sourcefilePath, destinationFolder);
SortResults srtResults = new SortResults(sourcefilePath, srtVars.DestFolder, srtVars.DbConnPath);
SortProgress srtProgress = new SortProgress();
try
{
srtResults.DeleteDuplicatesFile();
int lineCount = 1;
using (StreamReader reader = new StreamReader(sourcefilePath))
using (SqliteSortDefBulkInserter sortBulkInserter = new SqliteSortDefBulkInserter(srtVars.DbConnPath, sortDefinitions, maxBatchSize))
{
string line;
srtVars.Header = GetHeader(hasHeader, reader);
srtProgress.InitReading();
while ((line = reader.ReadLine()) != null)
{
srtResults.IncrementLinesRead();
ReportReadProgress(progress, srtProgress, srtResults.LinesRead);
if (dataFilter == null || dataFilter(line))
{
string[] keyValues = new string[sortDefinitions.GetKeys().Count];
setKeys(line, keyValues);
sortBulkInserter.Add(new SortKeyData { KeyValues = keyValues, Data = (line + Constants.Common.PreserveCharacter).Compress() });
lineCount++;
}
else
{
srtResults.IncrementFiltered();
}
}
sortBulkInserter.InsertAnyLeftOvers();
sortBulkInserter.AddUnUniqueIndex();
}
srtProgress.InitWriting();
if (writeOutSortFile)
{
srtResults.WriteOutSorted(dbConnPath: srtVars.DbConnPath,
header: srtVars.Header,
sortDefinitions: sortDefinitions,
delimiter: Constants.Delimiters.Tab,
returnDuplicates: returnDuplicates,
dupesFilePath: srtResults.DuplicatesFilePath,
compressed: true,
progress: (counter) => { srtProgress.Counter = counter; if (progress != null) { progress(srtProgress); } },
dataTransportation: dataTransportation,
deleteDb: deleteDbConnPath);
}
else
{
srtResults.Header = srtVars.Header;
}
srtResults.DeleteDuplicatesFileIfNoDuplicates();
}
catch (Exception)
{
CleanUp(srtVars, srtResults);
srtProgress = null;
throw;
}
return srtResults;
}
