public void CSVIntoMemDest(int numberOfRows, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destNonGeneric = new MemoryDestination();
var sourceGeneric = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destGeneric = new MemoryDestination <CSVData>();
var sourceDynamic = new CsvSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destDynamic = new MemoryDestination <ExpandoObject>();
sourceNonGeneric.ReleaseGCPressureRowCount = 500;
sourceGeneric.ReleaseGCPressureRowCount = 500;
sourceDynamic.ReleaseGCPressureRowCount = 500;
//Act
var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
var teGeneric = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
var teDynamic = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);
//Assert
Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
Assert.Equal(numberOfRows, destGeneric.Data.Count);
Assert.Equal(numberOfRows, destDynamic.Data.Count);
Assert.True(new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
}
public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "CsvDestinationWithTransformation");
var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var trans = new RowTransformation <ExpandoObject, CSVData>(
row =>
{
dynamic r = row as ExpandoObject;
return(new CSVData()
{
Col1 = r.Col1,
Col2 = r.Col2,
Col3 = r.Col3,
Col4 = r.Col4
});
});
var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize);
sourceExpando.LinkTo(trans);
trans.LinkTo(destGeneric);
//Act
long memAfter = 0;
long memBefore = 0;
bool startCheck = true;
int count = 1;
destGeneric.AfterBatchWrite = data =>
{
if (count++ % 50 == 0)
{
using Process proc = Process.GetCurrentProcess();
memAfter = proc.WorkingSet64;
if (startCheck)
{
memBefore = memAfter;
startCheck = false;
}
Assert.True(memAfter < (memBefore + (memBefore * deviation)));
}
};
var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
() =>
{
sourceExpando.Execute();
destGeneric.Wait();
}
);
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation).");
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation"));
//10.000.000 rows, batch size 10.000: ~8 min
//10.000.000 rows, batch size 1.000: ~10 min 10 sec
}
public void CompareGenericAndDynamic(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "CsvDestinationNonGenericETLBox");
ReCreateDestinationTable(connection, "CsvDestinationBulkInsert");
ReCreateDestinationTable(connection, "CsvDestinationGenericETLBox");
var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destNonGeneric = new DbDestination(connection, "CsvDestinationNonGenericETLBox", batchSize);
var sourceGeneric = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationGenericETLBox", batchSize);
//Act
var timeElapsedETLBoxNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
var timeElapsedETLBoxGeneric = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationNonGenericETLBox"));
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationGenericETLBox"));
Assert.True(Math.Abs(timeElapsedETLBoxGeneric.TotalMilliseconds - timeElapsedETLBoxNonGeneric.TotalMilliseconds) <
Math.Min(timeElapsedETLBoxGeneric.TotalMilliseconds, timeElapsedETLBoxNonGeneric.TotalMilliseconds) * deviation);
}
public void WriteParallelWhileTransactionOpen(IConnectionManager connection, int numberOfRows)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "TransactionDestination1");
ReCreateDestinationTable(connection, "TransactionDestination2");
var source = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var dest1 = new DbDestination(connection, "TransactionDestination1");
var dest2 = new DbDestination(connection, "TransactionDestination2");
var multi = new Multicast();
//Act & Assert
Assert.ThrowsAny <Exception>(() =>
{
connection.BeginTransaction();
source.LinkTo(multi);
multi.LinkTo(dest1);
multi.LinkTo(dest2);
source.Execute();
dest1.Wait();
dest2.Wait();
});
}
