public int Read(ImportFileOptions importTask, ISqlWriter sqlWriter)
{
Log.Debug($"ReadCsv for '{importTask.file}'");
Stopwatch _timer = new Stopwatch();
_timer.Start();
List <List <string> > batchLineFields = new List <List <string> >();
var fileInfo = new System.IO.FileInfo(importTask.file);
if (!System.IO.File.Exists(importTask.file))
{
Log.Error($"ReadCSV : '{importTask.file}' not exists!");
return(-1);
}
var firstLine = ReadFirstLine(fileInfo);
Char delimiter = string.IsNullOrWhiteSpace(importTask.delimiter) ? GuessDelimeter(firstLine) : importTask.delimiter.First <char>();
//Char quotingCharacter = '\0'; // no quoting-character;
Char quotingCharacter = string.IsNullOrWhiteSpace(importTask.quoting) ? GuessquotingCharacter(firstLine) : importTask.quoting.First <char>();
var rowCounter = 0;
try
{
using (var reader = new System.IO.StreamReader(fileInfo.FullName, Encoding.Default))
{
Char escapeCharacter = quotingCharacter;
using (var csv = new CsvReader(reader, false, delimiter, quotingCharacter, escapeCharacter, '\0', ValueTrimmingOptions.All))
{
csv.DefaultParseErrorAction = ParseErrorAction.ThrowException;
csv.SkipEmptyLines = true;
var headers = ReadHeaders(csv);
Log.Debug("Read:Headers : " + string.Join(",", headers));
sqlWriter.Init(importTask, headers);
if (!sqlWriter.EnsureFileIsUnique(fileInfo.Length))
{
Log.Debug($"Read: Import failed. The File '{importTask.file}' with size:'{fileInfo.Length}' has been allready imported.");
sqlWriter.UpdateStatusTable(-1, new TimeSpan(), fileInfo.Length);
return(-2);
}
do
{
// Read a pies of CSV
batchLineFields = ReadNextBatch(csv, importTask.batchSize);
if (batchLineFields.Count == 0)
{
break;
}
Log.Debug($"Read next batch : {batchLineFields.Count} entries start at {rowCounter} to {batchLineFields.Count + rowCounter}");
//Create & Write SQL
sqlWriter.Write(batchLineFields);
rowCounter += batchLineFields.Count;
} while (batchLineFields.Count > 0);
}
}
}
catch (Exception ex)
{
Log.Error($"ReadCsv catch exception : {ex.Message}");
return(-1);
}
_timer.Stop();
var timeSpan = TimeSpan.FromMilliseconds(_timer.ElapsedMilliseconds);
Log.Debug($"Import from '{importTask.file}' lenght:'{fileInfo.Length}' takes {_timer.ElapsedMilliseconds / 1000.0} seconds or {timeSpan.ToString("c")} of time. {rowCounter} rows inserted.");
sqlWriter.UpdateStatusTable(rowCounter, timeSpan, fileInfo.Length);
sqlWriter.ExecuteAdditionalSql();
return(rowCounter);
}
