private void ProcessRecordsExcel(object x) { ConcurrentExcelReader reader = x as ConcurrentExcelReader; SimpleSqlTableLoader writer = new SimpleSqlTableLoader(m_Context); string[] line; int rowsProcessed = 0; int numColumns = m_Context.ColumnNames.Count(); while (reader.TryExtractRecord(out line)) { var newRow = writer.GetEmptyRow(); for (int i = 0; i < numColumns; i++) { newRow[i] = line[i]; } writer.PostRecord(newRow); if (++rowsProcessed % numberOfLines == 0) { OnRecordsProcessed(Thread.CurrentThread.Name); } } //flush final records and trigger last event writer.WriteRecords(); OnRecordsProcessed(Thread.CurrentThread.Name, rowsProcessed % numberOfLines); }
public void WriteDbNullValuesTest() { var cont = new PipelineContext() { FirstLineContainsHeaders = false, SourceFilePath = @"..\..\..\D2S.LibraryTests\TestDataFiveColumns.txt", Delimiter = "||", TotalObjectsInSequentialPipe = 10, DestinationTableName = "dbo.WriteDbNullValuesTest", DbNullStringValue = "nullerino" }; DestinationTableCreator table = new DestinationTableCreator(cont); table.CreateTable(); SimpleSqlTableLoader loader = new SimpleSqlTableLoader(cont); var row = loader.GetEmptyRow(); for (int j = 0; j < cont.ColumnNamesSelection.Count(); j++) { row[j] = "nullerino"; } loader.PostRecord(row); try { loader.WriteRecords(); } finally { var val = ReadScalar(1, 1, "dbo.WriteDbNullValuesTest"); Assert.AreEqual(expected: DBNull.Value, actual: val); DropTableAndReturnRows("dbo.WriteDbNullValuesTest"); } }
public void WriteRecordNoHeaderTest() { var cont = new PipelineContext() { FirstLineContainsHeaders = false, SourceFilePath = @"..\..\..\D2S.LibraryTests\TestDataFiveColumns.txt", Delimiter = "||", TotalObjectsInSequentialPipe = 10, DestinationTableName = "dbo.SimpleSqlTableLoaderTest" }; DestinationTableCreator table = new DestinationTableCreator(cont); table.CreateTable(); SimpleSqlTableLoader loader = new SimpleSqlTableLoader(cont); for (int i = 0; i < 5; i++) { var row = loader.GetEmptyRow(); for (int j = 0; j < cont.ColumnNamesSelection.Count(); j++) { row[j] = $"col{j}"; } loader.PostRecord(row); } try { loader.WriteRecords(); } finally { int rowCount = DropTableAndReturnRows(cont.DestinationTableName); Assert.IsTrue(rowCount == 5); } }
public void WriteRecordsTest() { DestinationTableCreator destinationTableCreator = new DestinationTableCreator(context); destinationTableCreator.CreateTable(); SimpleSqlTableLoader loader = new SimpleSqlTableLoader(context); int numberOfRows = 100; for (int i = 0; i < numberOfRows; i++) { var row = loader.GetEmptyRow(); foreach (var column in context.ColumnNames) { row[column] = $"Value{i}"; } loader.PostRecord(row); } loader.WriteRecords(); int rowCount = DropTableAndReturnRows(context.DestinationTableName); Assert.AreEqual(expected: numberOfRows, actual: rowCount); }
private void ProcessRecords(object x) { ConcurrentFlatFileExtractor reader = x as ConcurrentFlatFileExtractor; SimpleSqlTableLoader writer = new SimpleSqlTableLoader(m_Context); string line; int rowsProcessed = 0; int numColumns = m_Context.ColumnNames.Count(); //if a selection is made on the source columns we will compute the ordinal rankings we require here int[] ordinalRankings = null; //if these are not equal a selection is made. if (numColumns != m_Context.ColumnNamesSelection.Count()) { ordinalRankings = new int[m_Context.ColumnNamesSelection.Count()]; int indexRankings = 0; //for every name in the total list, check if it is present in the selection and if so write its ordinal ranking to the array. //the rankings will be sorted low to high by design which also suits the simplesqlWriter in case it is in ordinal mode. for (int i = 0; i < numColumns; i++) { if (m_Context.ColumnNamesSelection.Any( selectedName => selectedName.Equals(m_Context.ColumnNames[i], StringComparison.InvariantCultureIgnoreCase))) { ordinalRankings[indexRankings++] = i; } } } while (reader.TryExtractLine(out line)) { string[] record = StringAndText.SplitRow(line, m_Context.Delimiter, m_Context.Qualifier, true); //assume the orindal rankings are identical (if all the pieces use the context.columnsnames property that will be the case //check the column count tho if (record.Count() != numColumns) { var errorMsg = $"A row was skipped over because it had too many or too few columns, expected: {numColumns}, actual: {record.Count()}"; if (m_Context.IsSkippingError) { LogService.Instance.Warn(errorMsg); } else { Exception ex = new Exception(errorMsg); LogService.Instance.Error(ex); throw ex; } } else { var newRow = writer.GetEmptyRow(); //write all columns if (ordinalRankings == null) { for (int i = 0; i < numColumns; i++) { newRow[i] = record[i]; } } //else write only selected columns (the indices we want are in the ordinalrankings array) else { for (int i = 0; i < ordinalRankings.Count(); i++) { newRow[i] = record[ordinalRankings[i]]; } } writer.PostRecord(newRow); if (++rowsProcessed % numberOfLines == 0) { OnRecordsProcessed(Thread.CurrentThread.Name); } } } //flush final records and trigger last event writer.WriteRecords(); OnRecordsProcessed(Thread.CurrentThread.Name, rowsProcessed % numberOfLines); }