private IEnumerable <object[]> GetRows(TextFieldParserBuffer parser) { for (int i = 0; (!_takeLines.HasValue || i < _takeLines.Value) && !parser.EndOfData; i++) { yield return(parser.ReadFields()); } }
private void Import() { var filePath = GetFilePath(); using var stream = File.OpenRead(filePath); using var bufferedStream = new BufferedStream(stream); using var parser = NewParser(bufferedStream, _separator.ToString()); var parserBuffer = new TextFieldParserBuffer(parser); // skip the specified number of initial file lines for (int i = 0; i < _skipLines && !parserBuffer.EndOfData; i++) { parserBuffer.SkipLine(); } SqlUtil.Import( ReadColumnNames(parserBuffer), GetRows(parserBuffer), _stmt.ImportTable, _temporaryTable, _truncateExistingTable, false, // stopAtFirstBlankRow _ifConversionFails, _blankValuesMethod, _notebook, _runner, _env); }
private List <string> ReadColumnNames(TextFieldParserBuffer parser) { var srcColNames = new List <string>(); if (_headerRow) { // read the column header row var cells = parser.ReadFields(); if (cells == null) { // end of file; there is nothing here. if (_skipLines == 0) { throw new Exception("No column header row was found because the file is empty."); } else { throw new Exception("No column header row was found because all rows were skipped."); } } // add a numeric suffix to each column name if necessary to make them all unique foreach (var cell in cells) { var testName = cell; var testNum = 1; while (srcColNames.Contains(testName)) { testNum++; testName = $"{cell}{testNum}"; } srcColNames.Add(testName); } } else { // no header row so use "column1", "column2", etc. based on the first row of data var fields = parser.PeekFields(); if (fields == null || !fields.Any()) { // treat empty file as a single column with no rows srcColNames.Add("column1"); } else { for (int i = 0; i < fields.Length; i++) { srcColNames.Add($"column{i + 1}"); } } } return(srcColNames); }
private void Import() { var filePath = GetFilePath(); using (var stream = File.OpenRead(filePath)) using (var bufferedStream = new BufferedStream(stream)) using (var parser = NewParser(bufferedStream)) { var parserBuffer = new TextFieldParserBuffer(parser); // skip the specified number of initial file lines for (int i = 0; i < _skipLines && !parserBuffer.EndOfData; i++) { parserBuffer.SkipLine(); } // consume the header row if present. invent generic column names if not. var srcColNames = ReadColumnNames(parserBuffer); // if the user specified a column list, then check to make sure all of the specified column names exist // in the list we just read. indices in dstColNodes/dstColNames match up with srcColNames, and some // elements of dstColNodes may be null if the user does not want to include certain CSV columns. // all dstColNodes elements will be null if the user specified no columns explicitly, which means they // want to include all columns. Ast.ImportColumn[] dstColNodes; string[] dstColNames; SqlUtil.GetDestinationColumns(_stmt.ImportTable.ImportColumns, _runner, _env, srcColNames, out dstColNodes, out dstColNames); // create or truncate the target table, if necessary. var dstTableName = _runner.EvaluateIdentifierOrExpr(_stmt.ImportTable.TableName, _env); SqlUtil.CreateOrTruncateTable(srcColNames, dstColNodes, dstColNames, dstTableName, _temporaryTable, _truncateExistingTable, _notebook); // ensure that the target table has all of the requested column names. SqlUtil.VerifyColumnsExist(dstColNames, dstTableName, _notebook); // read the data rows and insert into the target table. SqlUtil.InsertDataRows(GetRows(parserBuffer), dstColNames, dstColNodes, dstTableName, _ifConversionFails, _notebook); } }