public void TestMultipleBatches()
{
DataTable dt = new DataTable();
dt.Columns.Add("Col1");
dt.Columns.Add("Col2", typeof(int));
dt.Rows.Add("Fish", 123);
dt.Rows.Add("Fish", 123);
DataTable dt2 = new DataTable();
dt2.Columns.Add("Col1");
dt2.Columns.Add("Col2", typeof(int));
dt2.Rows.Add("Fish", 123);
dt2.Rows.Add("Haddock", 123);
var remover = new RemoveDuplicates();
//send it the batch with the duplication it will return 1 row
Assert.AreEqual(1, remover.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()).Rows.Count);
//now send it the second batch which contains 2 records, one duplication against first batch and one new one, expect only 1 row to come back
Assert.AreEqual(1, remover.ProcessPipelineData(dt2, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()).Rows.Count);
}
/// <summary>
/// Makes the current batch ONLY distinct. This only works if you have a bounded batch (see OrderByAndDistinctInMemory)
/// </summary>
/// <param name="chunk"></param>
/// <param name="listener"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private DataTable MakeDistinct(DataTable chunk, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
var removeDuplicates = new RemoveDuplicates()
{
NoLogging = true
};
return(removeDuplicates.ProcessPipelineData(chunk, listener, cancellationToken));
}
public void TestNulls()
{
DataTable dt = new DataTable();
dt.Columns.Add("Col1");
dt.Columns.Add("Col2", typeof(int));
dt.Rows.Add("Fish", 123);
dt.Rows.Add("Fish", null);
dt.Rows.Add(null, 123);
dt.Rows.Add("Pizza", null);
dt.Rows.Add(null, null);
dt.Rows.Add(null, null);
var remover = new RemoveDuplicates();
Assert.AreEqual(6, dt.Rows.Count);
//send it the batch with the duplication it will return 5 rows (the only duplicate is the double null)
Assert.AreEqual(5, remover.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()).Rows.Count);
}
