public void TryExtractRecordTest() { try { CreateTestTable(); ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context); BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>(); Action action = () => { object[] currObject = null; while (reader.TryExtractRecord(out currObject)) { output.TryAdd(currObject); } }; List <Task> tasks = new List <Task>(); for (int i = 0; i < 3; i++) { tasks.Add(Task.Factory.StartNew(action)); } Task.WhenAll(tasks).Wait(); Assert.IsTrue(output.Count == 5); } finally { DestroyTestTable(); } }
public void DbNullValueTest() { string OmegaLul = "HoHoHaHa"; try { CreateTestTable(); UpdateTestTable(); PipelineContext context = new PipelineContext() { SourceTableName = "dbo.ConcurrentSqlExtractorTest" , DbNullStringValue = OmegaLul }; ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context); BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>(); object[] row = null; while (reader.TryExtractRecord(out row)) { output.TryAdd(row); } Assert.IsTrue(output.Count == 5); object[] val = null; output.TryTake(out val); Assert.AreEqual(expected: OmegaLul, actual: val[0]); } finally { DestroyTestTable(); } }
public override async Task StartAsync() { //handle some pre-start conditions OutputToConsoleAndLog("Pipeline is starting"); //create shared objects ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(m_Context); ConcurrentFlatFileWriter writer = new ConcurrentFlatFileWriter(m_Context); Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string> state = new Tuple <ConcurrentSqlExtractor, ConcurrentFlatFileWriter, string>(reader, writer, m_Context.Delimiter); int NumTasks = m_Context.CpuCountUsedToComputeParallalism; Task[] tasks = new Task[NumTasks]; OutputToConsoleAndLog($"Creating {NumTasks} tasks"); //create tasks, to future editors, ensure that the state object is the correct type or create a new processrecords method for (int i = 0; i < NumTasks; i++) { tasks[i] = new Task(new Action <object>(ProcessRecords), state); } //write header line if needed if (m_Context.FirstLineContainsHeaders) { WriteHeaderLine(reader, writer); } OutputToConsoleAndLog("Starting tasks"); //start tasks for (int i = 0; i < NumTasks; i++) { tasks[i].Start(); } OutputToConsoleAndLog("Awaiting tasks"); //await the completion of asynchronous tasks await Task.WhenAll(tasks); writer.Close(); OutputToConsoleAndLog($"{ NumTasks} have finished completing. Pipeline finished"); }
private void WriteHeaderLine(ConcurrentSqlExtractor reader, ConcurrentFlatFileWriter writer) { var HeaderLineObject = reader.GetDataTable(); StringBuilder builder = new StringBuilder(); int count = HeaderLineObject.Rows.Count; string delim = m_Context.Delimiter; for (int i = 0; i < count - 1; i++) { builder.Append(HeaderLineObject.Rows[i]["ColumnName"]); builder.Append(delim); } builder.Append(HeaderLineObject.Rows[count - 1]["ColumnName"]); writer.WriteLine(builder.ToString()); }
public void GetDataTableTest() { try { CreateTestTable(); ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context); var result = reader.GetDataTable(); Assert.AreEqual(expected: 3, actual: result.Rows.Count); Assert.AreEqual(expected: "col1", actual: result.Rows[0]["ColumnName"]); } finally { DestroyTestTable(); } }
public void SkipperinoCappucinoMachiatoTest() { try { CreateTestTable(); ConcurrentSqlExtractor reader = new ConcurrentSqlExtractor(context); BoundedConcurrentQueu <object[]> output = new BoundedConcurrentQueu <object[]>(); object[] row = null; Assert.IsTrue(reader.TrySkipRecord()); Assert.IsTrue(reader.TrySkipRecord()); while (reader.TryExtractRecord(out row)) { output.TryAdd(row); } Assert.IsTrue(output.Count == 3); } finally { DestroyTestTable(); } }