static void Main(string[] args)
{
DataMoverLog.InfoAsync("Starting the application");
MigrationConfiguration configuration = null;
if (args.Length > 0)
{
DataMoverLog.DebugAsync("Received parameters:");
for (var i = 0; i < args.Length; i++)
{
DataMoverLog.DebugAsync($"{i} - {args[i]}");
}
configuration = ConfigurationLoader.LoadConfiguration(args);
}
if (configuration == null)
{
DataMoverLog.ErrorAsync("Missing configuration.");
PrintUsage();
}
else
{
var stream = System.IO.File.Open(configuration.Files[0].Path, FileMode.Open, FileAccess.Read,
FileShare.Read);
var loader = new CsvLoader(configuration.Files[0].Name);
foreach (var l in loader.ReadLines(stream))
{
}
}
Task.Delay(TimeSpan.FromMilliseconds(500)).Wait();
DataMoverLog.Terminate();
}
public IEnumerable <DataRow> ReadLines(Stream source)
{
DataMoverLog.DebugAsync($"Ready to load file \"{_name}\"");
_reader = new StreamReader(source);
_readerTask = Task.Run(() => FastReadLineTask());
var splitterTasks = Environment.ProcessorCount;
splitterTasks /= 2;
if (splitterTasks < 1)
{
splitterTasks = 1;
}
_splitterTasks = new Task[splitterTasks];
for (var i = 0; i < splitterTasks; i++)
{
_splitterTasks[i] = Task.Run(() => SplitIntoColumnsTask());
}
while (!_eof || _buffer.Count > 0)
{
while (_buffer.TryDequeue(out DataRow line))
{
yield return(line);
}
if (!_eof)
{
//DataMoverLog.DebugAsync("read buffer is empty");
Task.Delay(TimeSpan.FromMilliseconds(10)).Wait();
}
}
DataMoverLog.DebugAsync($"File \"{_name}\" is loaded.");
}
public static MigrationConfiguration ParseConfiguration(string configJson)
{
DataMoverLog.DebugAsync("Deserializing configuration string");
var config = JsonConvert.DeserializeObject <MigrationConfiguration>(configJson);
DataMoverLog.DebugAsync("Deserialized configuration string");
return(config);
}
public static MigrationConfiguration LoadConfiguration(string[] commandLineArgs)
{
if (commandLineArgs.Length > 0 &&
commandLineArgs[0].StartsWith("/c:", StringComparison.CurrentCultureIgnoreCase))
{
var config = LoadConfiguration(commandLineArgs[0].Substring(3));
if (config != null)
{
for (var i = 1; i < commandLineArgs.Length; i++)
{
var arg = commandLineArgs[i];
if (arg.StartsWith("/f:"))
{
var file = arg.Substring(3).Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
if (file.Length == 2)
{
if (config.Files == null)
{
config.Files = new List <File>();
var f = config.Files.FirstOrDefault(x =>
x.Name.Equals(file[0], StringComparison.CurrentCultureIgnoreCase));
if (f == null)
{
f = new File {
Name = file[0]
};
config.Files.Add(f);
}
f.Path = file[1];
}
}
else
{
DataMoverLog.ErrorAsync($"Parameter \"{arg}\" is not valid.");
}
}
else
{
DataMoverLog.ErrorAsync($"Unknown application parameter \"{arg}\"");
}
}
if (ValidateConfiguration(config))
{
return(config);
}
}
}
return(null);
}
private static bool ValidateConfiguration(MigrationConfiguration config)
{
DataMoverLog.DebugAsync("Validating configuration...");
foreach (var source in config.Sources)
{
if (!config.Files.Any(x => x.Name.Equals(source.Name, StringComparison.CurrentCultureIgnoreCase)))
{
DataMoverLog.ErrorAsync($"Configuration source \"{source.Name}\" does not have corresponding file path.");
return(false);
}
}
DataMoverLog.DebugAsync("Configuration seems to be ok.");
return(true);
}
private void SplitIntoColumnsTask()
{
DataMoverLog.DebugAsync($"Started line split thread for \"{_name}\"");
while (!_eof || _buffer.Count > 0 || _rawLineBuffer.Count > 0)
{
while (_rawLineBuffer.TryDequeue(out RawLine line))
{
var columns = this.SplitIntoColumns(line.Line, line.LineLength);
while (_bufferSizeRows != 0 && _buffer.Count > _bufferSizeRows)
{
Task.Delay(TimeSpan.FromMilliseconds(10)).Wait();
}
_buffer.Enqueue(new DataRow {
Columns = columns, RowNumber = line.LineNumber
});
}
}
DataMoverLog.DebugAsync($"Stopped line split thread for \"{_name}\"");
}
public static MigrationConfiguration LoadConfiguration(string filePath)
{
if (System.IO.File.Exists(filePath))
{
DataMoverLog.InfoAsync($"Loading configuration from file \"{filePath}\"");
try
{
var configJson = System.IO.File.ReadAllText(filePath);
DataMoverLog.InfoAsync($"Done reading configuration from file \"{filePath}\"");
return(ParseConfiguration(configJson));
}
catch (Exception ex)
{
DataMoverLog.ErrorAsync(ex.Message);
}
}
else
{
DataMoverLog.ErrorAsync($"File \"{filePath}\" does not exists");
}
return(null);
}
private unsafe void FastReadLineTask()
{
DataMoverLog.DebugAsync($"Started reader thread for \"{_name}\"");
try
{
var cBuff = new char[_readBufferSize];
fixed(char *bPtr = cBuff)
{
long counter = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize);
var bufferPosition = 0;
var lastelapsed = 0d;
while (!_reader.EndOfStream || bufferPosition < bufferLength)
{
bool hasLine = false;
int qualifierCount = 0;
char[] line = null;
var startPosition = bufferPosition;
var lineLength = 0;
void appendToLine()
{
if (MAX_LINE_SIZE < lineLength + bufferPosition - startPosition)
{
DataMoverLog.ErrorAsync($"Line {counter+1} in file \"{_name}\" is too big");
throw new FileLoadException("Line is too big");
}
var newLineLength = lineLength + bufferPosition - startPosition;
var newLine = new char[newLineLength];
if (lineLength > 0)
{
Array.Copy(line, newLine, lineLength);
}
Array.Copy(cBuff, startPosition, newLine, lineLength, bufferPosition - startPosition);
line = newLine;
lineLength = newLineLength;
}
while (!hasLine)
{
while (bufferPosition < bufferLength && ((qualifierCount > 0 && qualifierCount % 2 != 0) || (bPtr[bufferPosition] != '\r' && bPtr[bufferPosition] != '\n')))
{
if (bPtr[bufferPosition] == _textQualifier)
{
qualifierCount++;
}
bufferPosition++;
}
if (bufferPosition == bufferLength)
{
appendToLine();
if (_reader.EndOfStream)
{
break;
}
startPosition = 0;
bufferPosition = 0;
bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize);
}
else
{
if (bufferPosition > startPosition)
{
appendToLine();
}
if (lineLength > 0)
{
hasLine = true;
}
bufferPosition++;
startPosition = bufferPosition;
}
}
if (lineLength == 1 && (line[0] == '\r' || line[0] == '\n'))
{
continue;
}
counter++;
try
{
while (_bufferSizeRows != 0 && _rawLineBuffer.Count > _bufferSizeRows)
{
//DataMoverLog.DebugAsync($"Read buffer for {_name} is full");
Task.Delay(TimeSpan.FromMilliseconds(10)).Wait();
}
_rawLineBuffer.Enqueue(new RawLine {
Line = line, LineLength = lineLength, LineNumber = counter
});
} catch (Exception ex) {
DataMoverLog.ErrorAsync($"File \"{_name}\", row {counter} throws: {ex.Message}");
DataMoverLog.DebugAsync(new StringBuilder().Append(line).ToString());
}
if (stopwatch.Elapsed.TotalSeconds - lastelapsed >= 10)
{
DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds");
lastelapsed = stopwatch.Elapsed.TotalSeconds;
}
}
stopwatch.Stop();
DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds.");
_eof = true;
}
}catch (Exception ex) {
_eof = true;
DataMoverLog.ErrorAsync(ex.Message);
}
DataMoverLog.DebugAsync($"Stopped reader thread for \"{_name}\"");
}
