static void Main(string[] args) { DataMoverLog.InfoAsync("Starting the application"); MigrationConfiguration configuration = null; if (args.Length > 0) { DataMoverLog.DebugAsync("Received parameters:"); for (var i = 0; i < args.Length; i++) { DataMoverLog.DebugAsync($"{i} - {args[i]}"); } configuration = ConfigurationLoader.LoadConfiguration(args); } if (configuration == null) { DataMoverLog.ErrorAsync("Missing configuration."); PrintUsage(); } else { var stream = System.IO.File.Open(configuration.Files[0].Path, FileMode.Open, FileAccess.Read, FileShare.Read); var loader = new CsvLoader(configuration.Files[0].Name); foreach (var l in loader.ReadLines(stream)) { } } Task.Delay(TimeSpan.FromMilliseconds(500)).Wait(); DataMoverLog.Terminate(); }
public static MigrationConfiguration LoadConfiguration(string[] commandLineArgs) { if (commandLineArgs.Length > 0 && commandLineArgs[0].StartsWith("/c:", StringComparison.CurrentCultureIgnoreCase)) { var config = LoadConfiguration(commandLineArgs[0].Substring(3)); if (config != null) { for (var i = 1; i < commandLineArgs.Length; i++) { var arg = commandLineArgs[i]; if (arg.StartsWith("/f:")) { var file = arg.Substring(3).Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries); if (file.Length == 2) { if (config.Files == null) { config.Files = new List <File>(); var f = config.Files.FirstOrDefault(x => x.Name.Equals(file[0], StringComparison.CurrentCultureIgnoreCase)); if (f == null) { f = new File { Name = file[0] }; config.Files.Add(f); } f.Path = file[1]; } } else { DataMoverLog.ErrorAsync($"Parameter \"{arg}\" is not valid."); } } else { DataMoverLog.ErrorAsync($"Unknown application parameter \"{arg}\""); } } if (ValidateConfiguration(config)) { return(config); } } } return(null); }
private static bool ValidateConfiguration(MigrationConfiguration config) { DataMoverLog.DebugAsync("Validating configuration..."); foreach (var source in config.Sources) { if (!config.Files.Any(x => x.Name.Equals(source.Name, StringComparison.CurrentCultureIgnoreCase))) { DataMoverLog.ErrorAsync($"Configuration source \"{source.Name}\" does not have corresponding file path."); return(false); } } DataMoverLog.DebugAsync("Configuration seems to be ok."); return(true); }
public static MigrationConfiguration LoadConfiguration(string filePath) { if (System.IO.File.Exists(filePath)) { DataMoverLog.InfoAsync($"Loading configuration from file \"{filePath}\""); try { var configJson = System.IO.File.ReadAllText(filePath); DataMoverLog.InfoAsync($"Done reading configuration from file \"{filePath}\""); return(ParseConfiguration(configJson)); } catch (Exception ex) { DataMoverLog.ErrorAsync(ex.Message); } } else { DataMoverLog.ErrorAsync($"File \"{filePath}\" does not exists"); } return(null); }
private unsafe void FastReadLineTask() { DataMoverLog.DebugAsync($"Started reader thread for \"{_name}\""); try { var cBuff = new char[_readBufferSize]; fixed(char *bPtr = cBuff) { long counter = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); var bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize); var bufferPosition = 0; var lastelapsed = 0d; while (!_reader.EndOfStream || bufferPosition < bufferLength) { bool hasLine = false; int qualifierCount = 0; char[] line = null; var startPosition = bufferPosition; var lineLength = 0; void appendToLine() { if (MAX_LINE_SIZE < lineLength + bufferPosition - startPosition) { DataMoverLog.ErrorAsync($"Line {counter+1} in file \"{_name}\" is too big"); throw new FileLoadException("Line is too big"); } var newLineLength = lineLength + bufferPosition - startPosition; var newLine = new char[newLineLength]; if (lineLength > 0) { Array.Copy(line, newLine, lineLength); } Array.Copy(cBuff, startPosition, newLine, lineLength, bufferPosition - startPosition); line = newLine; lineLength = newLineLength; } while (!hasLine) { while (bufferPosition < bufferLength && ((qualifierCount > 0 && qualifierCount % 2 != 0) || (bPtr[bufferPosition] != '\r' && bPtr[bufferPosition] != '\n'))) { if (bPtr[bufferPosition] == _textQualifier) { qualifierCount++; } bufferPosition++; } if (bufferPosition == bufferLength) { appendToLine(); if (_reader.EndOfStream) { break; } startPosition = 0; bufferPosition = 0; bufferLength = _reader.ReadBlock(cBuff, 0, _readBufferSize); } else { if (bufferPosition > startPosition) { appendToLine(); } if (lineLength > 0) { hasLine = true; } bufferPosition++; startPosition = bufferPosition; } } if (lineLength == 1 && (line[0] == '\r' || line[0] == '\n')) { continue; } counter++; try { while (_bufferSizeRows != 0 && _rawLineBuffer.Count > _bufferSizeRows) { //DataMoverLog.DebugAsync($"Read buffer for {_name} is full"); Task.Delay(TimeSpan.FromMilliseconds(10)).Wait(); } _rawLineBuffer.Enqueue(new RawLine { Line = line, LineLength = lineLength, LineNumber = counter }); } catch (Exception ex) { DataMoverLog.ErrorAsync($"File \"{_name}\", row {counter} throws: {ex.Message}"); DataMoverLog.DebugAsync(new StringBuilder().Append(line).ToString()); } if (stopwatch.Elapsed.TotalSeconds - lastelapsed >= 10) { DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds"); lastelapsed = stopwatch.Elapsed.TotalSeconds; } } stopwatch.Stop(); DataMoverLog.DebugAsync($"Loaded {counter} lines from \"{_name}\" in {stopwatch.Elapsed.TotalSeconds} seconds."); _eof = true; } }catch (Exception ex) { _eof = true; DataMoverLog.ErrorAsync(ex.Message); } DataMoverLog.DebugAsync($"Stopped reader thread for \"{_name}\""); }