public void WithBatchChanges(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 2)
{
BeforeBatchWrite = rowArray =>
{
rowArray[0][1] = "NewValue";
return(rowArray);
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.Equal(2, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'"));
Assert.Equal(1, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'"));
}
public void CSVIntoMemDest(int numberOfRows, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destNonGeneric = new MemoryDestination();
var sourceGeneric = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destGeneric = new MemoryDestination <CSVData>();
var sourceDynamic = new CsvSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destDynamic = new MemoryDestination <ExpandoObject>();
sourceNonGeneric.ReleaseGCPressureRowCount = 500;
sourceGeneric.ReleaseGCPressureRowCount = 500;
sourceDynamic.ReleaseGCPressureRowCount = 500;
//Act
var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
var teGeneric = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
var teDynamic = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);
//Assert
Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
Assert.Equal(numberOfRows, destGeneric.Data.Count);
Assert.Equal(numberOfRows, destDynamic.Data.Count);
Assert.True(new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
}
public void AfterBatchWrite(IConnectionManager connection)
{
//Arrange
bool wasExecuted = false;
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 1)
{
AfterBatchWrite = rowArray =>
{
Assert.True(rowArray.Length == 1);
wasExecuted = true;
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.True(wasExecuted);
}
static void Main(string[] args)
{
//Set up database
Preparation();
//Define the source
CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");
//Define the transformation
RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
row => new Order()
{
Item = row[1],
Quantity = int.Parse(row[2]) + int.Parse(row[3]),
Price = int.Parse(row[4]) / 100
});
//DbDestination needs a connection manager pointing to the right database
SqlConnectionManager connMan = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;");
//Define the destination
DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");
//Link & run flow
source.LinkTo(rowTrans);
rowTrans.LinkTo(dest);
source.Execute();
dest.Wait();
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "CsvDestinationWithTransformation");
var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var trans = new RowTransformation <ExpandoObject, CSVData>(
row =>
{
dynamic r = row as ExpandoObject;
return(new CSVData()
{
Col1 = r.Col1,
Col2 = r.Col2,
Col3 = r.Col3,
Col4 = r.Col4
});
});
var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize);
sourceExpando.LinkTo(trans);
trans.LinkTo(destGeneric);
//Act
long memAfter = 0;
long memBefore = 0;
bool startCheck = true;
int count = 1;
destGeneric.AfterBatchWrite = data =>
{
if (count++ % 50 == 0)
{
using Process proc = Process.GetCurrentProcess();
memAfter = proc.WorkingSet64;
if (startCheck)
{
memBefore = memAfter;
startCheck = false;
}
Assert.True(memAfter < (memBefore + (memBefore * deviation)));
}
};
var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
() =>
{
sourceExpando.Execute();
destGeneric.Wait();
}
);
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation).");
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation"));
//10.000.000 rows, batch size 10.000: ~8 min
//10.000.000 rows, batch size 1.000: ~10 min 10 sec
}
public void WithObjectErrorLinking()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void InitAction()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv");
//Act
int IdOffset = 0;
RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>()
{
TransformationFunc = row =>
{
row.Col1 += IdOffset;
return(row);
},
InitAction = () => IdOffset += 1
};
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CsvSource <string[]> source = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata =>
{
return(new MySimpleRow()
{
Col1 = int.Parse(csvdata[0]),
Col2 = csvdata[1]
});
});
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CompareFlowWithBulkInsert(IConnectionManager connection, int numberOfRows, int batchSize, double deviationGeneric, double deviationBulk)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "CsvDestinationNonGenericETLBox");
ReCreateDestinationTable(connection, "CsvDestinationBulkInsert");
ReCreateDestinationTable(connection, "CsvDestinationGenericETLBox");
var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destNonGeneric = new DbDestination(connection, "CsvDestinationNonGenericETLBox", batchSize);
var sourceGeneric = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationGenericETLBox", batchSize);
//Act
var timeElapsedBulkInsert = GetBulkInsertTime(connection, numberOfRows);
var timeElapsedETLBoxNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
var timeElapsedETLBoxGeneric = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationNonGenericETLBox"));
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationGenericETLBox"));
Assert.True(Math.Abs(timeElapsedETLBoxGeneric.TotalMilliseconds - timeElapsedETLBoxNonGeneric.TotalMilliseconds) <
Math.Min(timeElapsedETLBoxGeneric.TotalMilliseconds, timeElapsedETLBoxNonGeneric.TotalMilliseconds) * deviationGeneric);
if (timeElapsedBulkInsert.TotalMilliseconds > 0)
{
Assert.True(timeElapsedBulkInsert < timeElapsedETLBoxNonGeneric);
Assert.True(timeElapsedBulkInsert.TotalMilliseconds * (deviationBulk + 1) > timeElapsedETLBoxNonGeneric.TotalMilliseconds);
}
}
public void MoreColumnsInDestination()
{
//Arrange
CreateTableTask.Create(SqlConnection, "CsvSourceDynamicColsInDest",
new List <TableColumn>()
{
new TableColumn("Col2", "VARCHAR(100)", allowNulls: true),
new TableColumn("Id", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "INT", allowNulls: true),
new TableColumn("ColX", "INT", allowNulls: true),
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInDest");
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 1 AND Col2='Test1' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 2 AND Col2='Test2' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 3 AND Col2='Test3' AND Id > 0 AND ColX IS NULL"));
}
static void Main(string[] args)
{
Preparation();
SqlConnectionManager connMan = new SqlConnectionManager("Data Source=.;Initial Catalog=demo;Integrated Security=false;User=sa;password=reallyStrongPwd123");
CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");
RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
row => new Order()
{
Id = int.Parse(row[0]),
Item = row[1],
Quantity = int.Parse(row[2]) + int.Parse(row[3]),
Price = double.Parse(row[4]) * 100
});
DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");
source.LinkTo(rowTrans);
rowTrans.LinkTo(dest);
source.Execute();
dest.Wait();
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic");
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
csvdata =>
{
dynamic c = csvdata as ExpandoObject;
c.Col1 = c.Header1;
c.Col2 = c.Header2;
return(c);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void NoHeadersFile_ParsesCorrectly()
{
var source = new CsvSource(_noHeadersFile, "false", ",");
DataTable table = source.GetData();
Assert.Equal(_csvLines.Split(',').Length, table.Columns.Count);
Assert.Equal(1, table.Rows.Count);
}
public void CsvSource_FullLoadTest()
{
var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, InterCommunicator.Empty);
var fired = source.Rows.Count();
Assert.AreEqual(11, fired);
}
private CsvSource <Poco> CreateDuplicateCsvSource(string fileName)
{
CsvSource <Poco> source = new CsvSource <Poco>(fileName);
source.Configuration.Delimiter = ";";
source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim;
source.Configuration.MissingFieldFound = null;
return(source);
}
public static void Run()
{
Console.WriteLine("Running data flow");
//Read data from csv file
CsvSource sourceOrderData = new CsvSource("DemoData.csv");
sourceOrderData.Configuration.Delimiter = ";";
//Transform into Order object
RowTransformation <ExpandoObject, Order> transIntoObject = new RowTransformation <ExpandoObject, Order>(
csvLine =>
{
dynamic order = csvLine as dynamic;
return(new Order()
{
//Header in Csv: OrderNumber;OrderItem;OrderAmount;CustomerName
Number = order.OrderNumber,
Item = order.OrderItem,
Amount = decimal.Parse(order.OrderAmount.ToString().Replace("€", ""), CultureInfo.GetCultureInfo("en-US")),
CustomerName = order.CustomerName
});
});
sourceOrderData.LinkTo(transIntoObject);
//Find corresponding customer id if customer exists in Customer table
DbSource <Customer> sourceCustomerData = new DbSource <Customer>("customer");
LookupTransformation <Order, Customer> lookupCustomerKey = new LookupTransformation <Order, Customer>(sourceCustomerData);
transIntoObject.LinkTo(lookupCustomerKey);
//Split data
Multicast <Order> multiCast = new Multicast <Order>();
lookupCustomerKey.LinkTo(multiCast);
//Store Order in Orders table
DbDestination <Order> destOrderTable = new DbDestination <Order>("orders");
multiCast.LinkTo(destOrderTable);
//Create rating for existing customers based total of order amount
Aggregation <Order, Rating> aggregation = new Aggregation <Order, Rating>();
multiCast.LinkTo(aggregation);
//Store the rating in the customer rating table
DbDestination <Rating> destRating = new DbDestination <Rating>("customer_rating");
aggregation.LinkTo(destRating);
//Execute the data flow synchronously
sourceOrderData.Execute();
destOrderTable.Wait();
destRating.Wait();
}
public void reads_data_from_csv_file_with_two_columns_into_analysis_data()
{
var file = Path.Join(BasePath, "TwoColumns.csv");
var csvSource = new CsvSource(file, ",");
csvSource.Data.XUnit.ShouldBe("x");
csvSource.Data.YUnit.ShouldBe("y");
csvSource.Data.PointSets.ShouldBe(new [] { new Point[] { (0.1, 1), (0.2, 2), (0.3, 3) } });
public void CsvSource_CancelledLoadTest()
{
var tokenSource = new CancellationTokenSource();
tokenSource.Cancel();
var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, new RuntimeContext(tokenSource.Token, new ISchemaColumn[0]));
var fired = source.Rows.Count();
Assert.AreEqual(0, fired);
}
public void CsvSource_CancelledLoadTest()
{
var tokenSource = new CancellationTokenSource();
tokenSource.Cancel();
var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, new InterCommunicator(tokenSource.Token));
var fired = source.Rows.Count();
Assert.AreEqual(0, fired);
}
static void Main(string[] args)
{
var source = new CsvSource <InputData>("Accounts_Quartal1.csv");
source.Configuration.Delimiter = ";";
var trans = new RowMultiplication <InputData, PivotedOutput>();
trans.MultiplicationFunc = row =>
{
List <PivotedOutput> result = new List <PivotedOutput>();
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.January),
MonthlyValue = row.January
});
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.February),
MonthlyValue = row.February
});
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.March),
MonthlyValue = row.March
});
return(result);
};
var dest = new CsvDestination <PivotedOutput>("AccountNumbers_Pivoted.csv");
dest.Configuration.HasHeaderRecord = false;
source.LinkTo(trans);
trans.LinkTo(dest);
Network.Execute(source);
/* AccountNumbers_Pivoted.csv output:
* 4711,January,10
* 4711,February,11
* 4711,March,12
* 4712,January,20
* 4712,February,21
* 4712,March,22
* 4713,January,30
* 4713,February,31
* 4713,March,32
*/
}
public void SplitCsvSourceIn2Tables()
{
//Arrange
TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("SplitDataDestination1");
FourColumnsTableFixture dest2Table = new FourColumnsTableFixture("SplitDataDestination2");
var source = new CsvSource <CSVPoco>("res/Multicast/CsvSourceToSplit.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Delimiter = ";"
}
};
var multicast = new Multicast <CSVPoco>();
var row1 = new RowTransformation <CSVPoco, Entity1>(input => {
return(new Entity1
{
Col1 = input.CSVCol1,
Col2 = input.CSVCol2
});
});
var row2 = new RowTransformation <CSVPoco, Entity2>(input => {
return(new Entity2
{
Col2 = input.CSVCol2,
Col3 = input.CSVCol3,
Col4 = input.CSVCol4
});
});
var destination1 = new DbDestination <Entity1>(Connection, "SplitDataDestination1");
var destination2 = new DbDestination <Entity2>(Connection, "SplitDataDestination2");
//Act
source.LinkTo(multicast);
multicast.LinkTo(row1);
multicast.LinkTo(row2);
row1.LinkTo(destination1);
row2.LinkTo(destination2);
source.Execute();
destination1.Wait();
destination2.Wait();
//Assert
dest1Table.AssertTestData();
dest2Table.AssertTestData();
}
public void HeadersFile_ParsesWithHeaderNames()
{
var source = new CsvSource(_headersFile, "true", ",");
DataTable table = source.GetData();
Assert.Equal(_csvLines.Split(',').Length, table.Columns.Count);
Assert.Equal(1, table.Rows.Count);
string[] headerNames = _headers.Split(',');
for (int idx = 0; idx < headerNames.Length; idx++)
{
Assert.Equal(headerNames[idx], table.Columns[idx].ColumnName);
}
}
public void SimpleFlowWithDynamicObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamic");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("CsvSourceDynamic", SqlConnection);
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithoutErrorLinking()
{
//Arrange
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");
//Assert
Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void MoreColumnsInCSV()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void MoreColumnsInDatabase()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/OneColumn.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "CsvDestination2Columns", "Col1 IN (1,2,3)"));
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSource2Cols");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSource2Cols", Connection);
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void IdentityAtTheEnd()
{
//Arrange
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CsvDestination4Columns", identityColumnIndex: 3);
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination4Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumnsNoId.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
public void MoreColumnsInSource()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamicColsInSource");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInSource");
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/FourColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void SimpleLookupWithDynamicObject(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SourceLookupDynamicObject");
source2Columns.InsertTestData();
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture(connection, "DestinationLookupDynamicObject", -1);
DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(connection, "SourceLookupDynamicObject");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(connection, "DestinationLookupDynamicObject");
//Act
List <ExpandoObject> lookupList = new List <ExpandoObject>();
CsvSource <ExpandoObject> lookupSource = new CsvSource <ExpandoObject>("res/Lookup/LookupSource.csv");
var lookup = new LookupTransformation <ExpandoObject, ExpandoObject>(
lookupSource,
row =>
{
dynamic r = row as ExpandoObject;
r.Col3 = lookupList
.Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
.Select(lkupRow =>
{
dynamic lk = lkupRow as dynamic;
return(lk.Column3 == string.Empty ? null : Int64.Parse(lk.Column3));
})
.FirstOrDefault();
r.Col4 = lookupList
.Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
.Select(lkupRow => { dynamic lk = lkupRow as dynamic; return(double.Parse(lk.Column4)); })
.FirstOrDefault();
return(row);
},
lookupList
);
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
public void CSVGenericWithSkipRows_DB()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceSkipRows");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceSkipRows", Connection);
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsSkipRows.csv");
source.SkipRows = 2;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CsvSourceNoHeader()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceNoHeader");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "CsvSourceNoHeader");
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsNoHeader.csv");
source.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void NonExistentFile_Throws()
{
var source = new CsvSource(@"c:\does\not\exist.csv", "false", ",");
Assert.Throws<ArgumentException>(() => source.GetData());
}
public void SayHasHeaders_NoHeaders_Throws()
{
var source = new CsvSource(_noHeadersFile, "true", ",");
Assert.Throws<Exception>(() => source.GetData());
}