static void Main(string[] args) { DataMoverLog.InfoAsync("Starting the application"); MigrationConfiguration configuration = null; if (args.Length > 0) { DataMoverLog.DebugAsync("Received parameters:"); for (var i = 0; i < args.Length; i++) { DataMoverLog.DebugAsync($"{i} - {args[i]}"); } configuration = ConfigurationLoader.LoadConfiguration(args); } if (configuration == null) { DataMoverLog.ErrorAsync("Missing configuration."); PrintUsage(); } else { var stream = System.IO.File.Open(configuration.Files[0].Path, FileMode.Open, FileAccess.Read, FileShare.Read); var loader = new CsvLoader(configuration.Files[0].Name); foreach (var l in loader.ReadLines(stream)) { } } Task.Delay(TimeSpan.FromMilliseconds(500)).Wait(); DataMoverLog.Terminate(); }
public IEnumerable <DataRow> ReadLines(Stream source) { DataMoverLog.DebugAsync($"Ready to load file \"{_name}\""); _reader = new StreamReader(source); _readerTask = Task.Run(() => FastReadLineTask()); var splitterTasks = Environment.ProcessorCount; splitterTasks /= 2; if (splitterTasks < 1) { splitterTasks = 1; } _splitterTasks = new Task[splitterTasks]; for (var i = 0; i < splitterTasks; i++) { _splitterTasks[i] = Task.Run(() => SplitIntoColumnsTask()); } while (!_eof || _buffer.Count > 0) { while (_buffer.TryDequeue(out DataRow line)) { yield return(line); } if (!_eof) { //DataMoverLog.DebugAsync("read buffer is empty"); Task.Delay(TimeSpan.FromMilliseconds(10)).Wait(); } } DataMoverLog.DebugAsync($"File \"{_name}\" is loaded."); }
public static MigrationConfiguration ParseConfiguration(string configJson) { DataMoverLog.DebugAsync("Deserializing configuration string"); var config = JsonConvert.DeserializeObject <MigrationConfiguration>(configJson); DataMoverLog.DebugAsync("Deserialized configuration string"); return(config); }
private static bool ValidateConfiguration(MigrationConfiguration config) { DataMoverLog.DebugAsync("Validating configuration..."); foreach (var source in config.Sources) { if (!config.Files.Any(x => x.Name.Equals(source.Name, StringComparison.CurrentCultureIgnoreCase))) { DataMoverLog.ErrorAsync($"Configuration source \"{source.Name}\" does not have corresponding file path."); return(false); } } DataMoverLog.DebugAsync("Configuration seems to be ok."); return(true); }
private void SplitIntoColumnsTask() { DataMoverLog.DebugAsync($"Started line split thread for \"{_name}\""); while (!_eof || _buffer.Count > 0 || _rawLineBuffer.Count > 0) { while (_rawLineBuffer.TryDequeue(out RawLine line)) { var columns = this.SplitIntoColumns(line.Line, line.LineLength); while (_bufferSizeRows != 0 && _buffer.Count > _bufferSizeRows) { Task.Delay(TimeSpan.FromMilliseconds(10)).Wait(); } _buffer.Enqueue(new DataRow { Columns = columns, RowNumber = line.LineNumber }); } } DataMoverLog.DebugAsync($"Stopped line split thread for \"{_name}\""); }
private unsafe void FastReadLineTask() { DataMoverLog.DebugAsync($"Started reader thread for \"{_name}\""); try { var cBuff = new char[_readBufferSize]; fixed(char *bPtr = cBuff) { long counter = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); var bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize); var bufferPosition = 0; var lastelapsed = 0d; while (!_reader.EndOfStream || bufferPosition < bufferLength) { bool hasLine = false; int qualifierCount = 0; char[] line = null; var startPosition = bufferPosition; var lineLength = 0; void appendToLine() { if (MAX_LINE_SIZE < lineLength + bufferPosition - startPosition) { DataMoverLog.ErrorAsync($"Line {counter+1} in file \"{_name}\" is too big"); throw new FileLoadException("Line is too big"); } var newLineLength = lineLength + bufferPosition - startPosition; var newLine = new char[newLineLength]; if (lineLength > 0) { Array.Copy(line, newLine, lineLength); } Array.Copy(cBuff, startPosition, newLine, lineLength, bufferPosition - startPosition); line = newLine; lineLength = newLineLength; } while (!hasLine) { while (bufferPosition < bufferLength && ((qualifierCount > 0 && qualifierCount % 2 != 0) || (bPtr[bufferPosition] != '\r' && bPtr[bufferPosition] != '\n'))) { if (bPtr[bufferPosition] == _textQualifier) { qualifierCount++; } bufferPosition++; } if (bufferPosition == bufferLength) { appendToLine(); if (_reader.EndOfStream) { break; } startPosition = 0; bufferPosition = 0; bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize); } else { if (bufferPosition > startPosition) { appendToLine(); } if (lineLength > 0) { hasLine = true; } bufferPosition++; startPosition = bufferPosition; } } if (lineLength == 1 && (line[0] == '\r' || line[0] == '\n')) { continue; } counter++; try { while (_bufferSizeRows != 0 && _rawLineBuffer.Count > _bufferSizeRows) { //DataMoverLog.DebugAsync($"Read buffer for {_name} is full"); Task.Delay(TimeSpan.FromMilliseconds(10)).Wait(); } _rawLineBuffer.Enqueue(new RawLine { Line = line, LineLength = lineLength, LineNumber = counter }); } catch (Exception ex) { DataMoverLog.ErrorAsync($"File \"{_name}\", row {counter} throws: {ex.Message}"); DataMoverLog.DebugAsync(new StringBuilder().Append(line).ToString()); } if (stopwatch.Elapsed.TotalSeconds - lastelapsed >= 10) { DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds"); lastelapsed = stopwatch.Elapsed.TotalSeconds; } } stopwatch.Stop(); DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds."); _eof = true; } }catch (Exception ex) { _eof = true; DataMoverLog.ErrorAsync(ex.Message); } DataMoverLog.DebugAsync($"Stopped reader thread for \"{_name}\""); }