private static string[] GetMasterFields <T>(MasterFixedWidthFileSource <T> master, string data)
{
string outLine = SortFileHelpers.UnEscapeByDelimiter(data.Decompress(), Constants.Delimiters.Tab);
string[] results = null;
FileParser.ParseFixedWidthString(new StringReader(outLine), (fields, lNum) =>
{
results = fields;
}, master.FixedWidths);
return(results);
}
private static string[] GetMasterFields <T>(MasterDelimitedFileSource <T> master, string data)
{
string outLine = SortFileHelpers.UnEscapeByDelimiter(data, master.Delimiter);
string[] results = null;
FileParser.ParseDelimitedString(new StringReader(outLine), (fields, lNum) =>
{
results = fields;
}, master.Delimiter);
return(results);
}
internal void WriteOutSorted(string dbConnPath,
string header,
SortDirection sortDir,
string delimiter = Constants.Delimiters.Comma,
bool hasUniqueIndex = false,
bool returnDuplicates = false,
string dupesFilePath = "",
bool compressed = false,
Action <int> progress = null,
DataTransportation dataTransportation = null,
bool deleteDb = true)
{
bool writeSortedFile = WriteoutSortedFile(dataTransportation);
if (writeSortedFile)
{
DeleteSortedFile();
}
this.Header = header;
StreamWriter dupeWriter = !string.IsNullOrEmpty(dupesFilePath) ? new StreamWriter(dupesFilePath) : null;
StreamWriter sw = writeSortedFile ? new StreamWriter(SortedFilePath) : null;
using (sw)
using (dupeWriter)
{
if (!string.IsNullOrWhiteSpace(header))
{
if (sw != null)
{
sw.WriteLine(header);
}
if (returnDuplicates)
{
WriteHeaderForDuplicatesFile(true, header, dupeWriter);
}
}
using (var cn = new SQLiteConnection(@"Data Source=" + dbConnPath))
{
string selectCmd = "SELECT * FROM FileData ORDER BY SortKey";
if (sortDir == SortDirection.Descending)
{
selectCmd += " DESC";
}
cn.Open();
using (var cmd = new SQLiteCommand(selectCmd, cn))
using (SQLiteDataReader rdr = cmd.ExecuteReader())
{
dynamic lastReadKey = null;
while (rdr.Read())
{
string sqlLiteData = (string)rdr["LineData"];
string sqlLiteoutLine = SortFileHelpers.UnEscapeByDelimiter(compressed ? sqlLiteData.Decompress() : sqlLiteData, delimiter);
if (hasUniqueIndex)
{
dynamic sqlLiteKey = rdr["SortKey"];
if (sqlLiteKey.Equals(lastReadKey))
{
if (returnDuplicates)
{
dupeWriter.WriteLine(sqlLiteoutLine);
this.IncrementDuplicates();
}
continue;
}
lastReadKey = sqlLiteKey;
}
if (sw != null)
{
sw.WriteLine(sqlLiteoutLine);
}
IncrementLinesSorted();
ReportProgress(progress, LinesSorted);
DoDataTransportPassthrough(dataTransportation, sqlLiteoutLine);
}
}
cn.Close();
}
}
if (deleteDb)
{
SortFileHelpers.DeleteFileIfExists(dbConnPath);
}
}
internal void WriteOutSorted(string dbConnPath,
string header,
SortDefinitions sortDefinitions,
string delimiter = Constants.Delimiters.Comma,
bool returnDuplicates = false,
string dupesFilePath = "",
bool compressed = false,
Action <int> progress = null,
DataTransportation dataTransportation = null,
bool deleteDb = true)
{
bool writeSortedFile = WriteoutSortedFile(dataTransportation);
if (writeSortedFile)
{
DeleteSortedFile();
}
this.Header = header;
StreamWriter dupeWriter = !string.IsNullOrEmpty(dupesFilePath) ? new StreamWriter(dupesFilePath) : null;
StreamWriter sw = writeSortedFile ? new StreamWriter(SortedFilePath) : null;
using (sw)
using (dupeWriter)
{
if (!string.IsNullOrWhiteSpace(header))
{
if (sw != null)
{
sw.WriteLine(header);
}
if (returnDuplicates)
{
WriteHeaderForDuplicatesFile(true, header, dupeWriter);
}
}
using (var cn = new SQLiteConnection(@"Data Source=" + dbConnPath))
{
string selectCmd = "SELECT * FROM FileData ORDER BY " + sortDefinitions.BuildOrderClause();
cn.Open();
using (var cmd = new SQLiteCommand(selectCmd, cn))
using (SQLiteDataReader rdr = cmd.ExecuteReader())
{
var lastReadKeyList = GetNewDynamicListForKeys(sortDefinitions);
while (rdr.Read())
{
string sqlLiteData = (string)rdr["LineData"];
string sqlLiteoutLine = SortFileHelpers.UnEscapeByDelimiter(compressed ? sqlLiteData.Decompress() : sqlLiteData, delimiter);
if (lastReadKeyList.Count > 0)
{
var currentReadKeyList = SetNewDynamicListForKeysValues(sortDefinitions, rdr);
if (KeysEqual(currentReadKeyList, lastReadKeyList))
{
if (returnDuplicates)
{
dupeWriter.WriteLine(sqlLiteoutLine);
this.IncrementDuplicates();
}
continue;
}
lastReadKeyList = currentReadKeyList;
}
if (sw != null)
{
sw.WriteLine(sqlLiteoutLine);
}
IncrementLinesSorted();
ReportProgress(progress, LinesSorted);
DoDataTransportPassthrough(dataTransportation, sqlLiteoutLine);
}
}
cn.Close();
}
}
if (deleteDb)
{
SortFileHelpers.DeleteFileIfExists(dbConnPath);
}
}