示例#1
0
        public void WithBatchChanges(IConnectionManager connection)
        {
            //Arrange
            TwoColumnsTableFixture   d2c  = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
            DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 2)
            {
                BeforeBatchWrite = rowArray =>
                {
                    rowArray[0][1] = "NewValue";
                    return(rowArray);
                }
            };

            //Act
            CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
            Assert.Equal(2, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'"));
            Assert.Equal(1, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'"));
        }
示例#2
0
        public void CSVIntoMemDest(int numberOfRows, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);

            var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destNonGeneric   = new MemoryDestination();
            var sourceGeneric    = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destGeneric      = new MemoryDestination <CSVData>();
            var sourceDynamic    = new CsvSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destDynamic      = new MemoryDestination <ExpandoObject>();

            sourceNonGeneric.ReleaseGCPressureRowCount = 500;
            sourceGeneric.ReleaseGCPressureRowCount    = 500;
            sourceDynamic.ReleaseGCPressureRowCount    = 500;
            //Act
            var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
            var teGeneric    = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
            var teDynamic    = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);

            //Assert
            Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
            Assert.Equal(numberOfRows, destGeneric.Data.Count);
            Assert.Equal(numberOfRows, destDynamic.Data.Count);
            Assert.True(new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
                        new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
        }
示例#3
0
        public void AfterBatchWrite(IConnectionManager connection)
        {
            //Arrange
            bool wasExecuted              = false;
            TwoColumnsTableFixture   d2c  = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
            DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 1)
            {
                AfterBatchWrite = rowArray =>
                {
                    Assert.True(rowArray.Length == 1);
                    wasExecuted = true;
                }
            };

            //Act
            CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
            Assert.True(wasExecuted);
        }
示例#4
0
        static void Main(string[] args)
        {
            //Set up database
            Preparation();

            //Define the source
            CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");

            //Define the transformation
            RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
                row => new Order()
            {
                Item     = row[1],
                Quantity = int.Parse(row[2]) + int.Parse(row[3]),
                Price    = int.Parse(row[4]) / 100
            });

            //DbDestination needs a connection manager pointing to the right database
            SqlConnectionManager connMan = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;");
            //Define the destination
            DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");

            //Link & run flow
            source.LinkTo(rowTrans);
            rowTrans.LinkTo(dest);
            source.Execute();
            dest.Wait();

            Console.WriteLine("Press any key to continue...");
            Console.ReadLine();
        }
示例#5
0
        public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
            ReCreateDestinationTable(connection, "CsvDestinationWithTransformation");

            var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var trans         = new RowTransformation <ExpandoObject, CSVData>(
                row =>
            {
                dynamic r = row as ExpandoObject;
                return(new CSVData()
                {
                    Col1 = r.Col1,
                    Col2 = r.Col2,
                    Col3 = r.Col3,
                    Col4 = r.Col4
                });
            });
            var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize);

            sourceExpando.LinkTo(trans);
            trans.LinkTo(destGeneric);

            //Act
            long memAfter   = 0;
            long memBefore  = 0;
            bool startCheck = true;
            int  count      = 1;

            destGeneric.AfterBatchWrite = data =>
            {
                if (count++ % 50 == 0)
                {
                    using Process proc = Process.GetCurrentProcess();
                    memAfter           = proc.WorkingSet64;
                    if (startCheck)
                    {
                        memBefore  = memAfter;
                        startCheck = false;
                    }
                    Assert.True(memAfter < (memBefore + (memBefore * deviation)));
                }
            };

            var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
                                                                   () =>
            {
                sourceExpando.Execute();
                destGeneric.Wait();
            }
                                                                   );

            output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation).");

            //Assert
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation"));
            //10.000.000 rows, batch size 10.000: ~8 min
            //10.000.000 rows, batch size  1.000: ~10 min 10 sec
        }
        public void WithObjectErrorLinking()
        {
            //Arrange
            TwoColumnsTableFixture          dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking");
            DbDestination <MySimpleRow>     dest         = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection);
            MemoryDestination <ETLBoxError> errorDest    = new MemoryDestination <ETLBoxError>();

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");

            source.LinkTo(dest);
            source.LinkErrorTo(errorDest);
            source.Execute();
            dest.Wait();
            errorDest.Wait();

            //Assert
            dest2Columns.AssertTestData();
            Assert.Collection <ETLBoxError>(errorDest.Data,
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
                                            );
        }
        public void InitAction()
        {
            //Arrange
            TwoColumnsTableFixture  dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
            CsvSource <MySimpleRow> source       = new CsvSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv");

            //Act
            int IdOffset = 0;
            RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>()
            {
                TransformationFunc = row =>
                {
                    row.Col1 += IdOffset;
                    return(row);
                },
                InitAction = () => IdOffset += 1
            };
            DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");

            source.LinkTo(trans);
            trans.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void ConvertIntoObject()
        {
            //Arrange
            TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
            CsvSource <string[]>   source       = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv");

            //Act
            RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
                csvdata =>
            {
                return(new MySimpleRow()
                {
                    Col1 = int.Parse(csvdata[0]),
                    Col2 = csvdata[1]
                });
            });
            DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");

            source.LinkTo(trans);
            trans.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
示例#9
0
        public void CompareFlowWithBulkInsert(IConnectionManager connection, int numberOfRows, int batchSize, double deviationGeneric, double deviationBulk)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
            ReCreateDestinationTable(connection, "CsvDestinationNonGenericETLBox");
            ReCreateDestinationTable(connection, "CsvDestinationBulkInsert");
            ReCreateDestinationTable(connection, "CsvDestinationGenericETLBox");

            var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destNonGeneric   = new DbDestination(connection, "CsvDestinationNonGenericETLBox", batchSize);
            var sourceGeneric    = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destGeneric      = new DbDestination <CSVData>(connection, "CsvDestinationGenericETLBox", batchSize);

            //Act
            var timeElapsedBulkInsert       = GetBulkInsertTime(connection, numberOfRows);
            var timeElapsedETLBoxNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
            var timeElapsedETLBoxGeneric    = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);


            //Assert
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationNonGenericETLBox"));
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationGenericETLBox"));
            Assert.True(Math.Abs(timeElapsedETLBoxGeneric.TotalMilliseconds - timeElapsedETLBoxNonGeneric.TotalMilliseconds) <
                        Math.Min(timeElapsedETLBoxGeneric.TotalMilliseconds, timeElapsedETLBoxNonGeneric.TotalMilliseconds) * deviationGeneric);
            if (timeElapsedBulkInsert.TotalMilliseconds > 0)
            {
                Assert.True(timeElapsedBulkInsert < timeElapsedETLBoxNonGeneric);
                Assert.True(timeElapsedBulkInsert.TotalMilliseconds * (deviationBulk + 1) > timeElapsedETLBoxNonGeneric.TotalMilliseconds);
            }
        }
        public void MoreColumnsInDestination()
        {
            //Arrange
            CreateTableTask.Create(SqlConnection, "CsvSourceDynamicColsInDest",
                                   new List <TableColumn>()
            {
                new TableColumn("Col2", "VARCHAR(100)", allowNulls: true),
                new TableColumn("Id", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
                new TableColumn("Col1", "INT", allowNulls: true),
                new TableColumn("ColX", "INT", allowNulls: true),
            });
            DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInDest");

            //Act
            CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(3, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest"));
            Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 1 AND Col2='Test1' AND Id > 0 AND ColX IS NULL"));
            Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 2 AND Col2='Test2' AND Id > 0 AND ColX IS NULL"));
            Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 3 AND Col2='Test3' AND Id > 0 AND ColX IS NULL"));
        }
示例#11
0
        static void Main(string[] args)
        {
            Preparation();

            SqlConnectionManager connMan = new SqlConnectionManager("Data Source=.;Initial Catalog=demo;Integrated Security=false;User=sa;password=reallyStrongPwd123");

            CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");

            RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
                row => new Order()
            {
                Id       = int.Parse(row[0]),
                Item     = row[1],
                Quantity = int.Parse(row[2]) + int.Parse(row[3]),
                Price    = double.Parse(row[4]) * 100
            });

            DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");

            source.LinkTo(rowTrans);
            rowTrans.LinkTo(dest);
            source.Execute();

            dest.Wait();

            Console.WriteLine("Press any key to continue...");
            Console.ReadLine();
        }
        public void ConvertIntoObject()
        {
            //Arrange
            TwoColumnsTableFixture    dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic");
            CsvSource <ExpandoObject> source       = new CsvSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv");

            //Act
            RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
                csvdata =>
            {
                dynamic c = csvdata as ExpandoObject;
                c.Col1    = c.Header1;
                c.Col2    = c.Header2;
                return(c);
            });
            DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic");

            source.LinkTo(trans);
            trans.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void NoHeadersFile_ParsesCorrectly()
        {
            var source = new CsvSource(_noHeadersFile, "false", ",");
            DataTable table = source.GetData();

            Assert.Equal(_csvLines.Split(',').Length, table.Columns.Count);
            Assert.Equal(1, table.Rows.Count);
        }
示例#14
0
        public void CsvSource_FullLoadTest()
        {
            var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, InterCommunicator.Empty);

            var fired = source.Rows.Count();

            Assert.AreEqual(11, fired);
        }
示例#15
0
        private CsvSource <Poco> CreateDuplicateCsvSource(string fileName)
        {
            CsvSource <Poco> source = new CsvSource <Poco>(fileName);

            source.Configuration.Delimiter         = ";";
            source.Configuration.TrimOptions       = CsvHelper.Configuration.TrimOptions.Trim;
            source.Configuration.MissingFieldFound = null;
            return(source);
        }
示例#16
0
        public static void Run()
        {
            Console.WriteLine("Running data flow");

            //Read data from csv file
            CsvSource sourceOrderData = new CsvSource("DemoData.csv");

            sourceOrderData.Configuration.Delimiter = ";";

            //Transform into Order object
            RowTransformation <ExpandoObject, Order> transIntoObject = new RowTransformation <ExpandoObject, Order>(
                csvLine =>
            {
                dynamic order = csvLine as dynamic;
                return(new Order()
                {
                    //Header in Csv: OrderNumber;OrderItem;OrderAmount;CustomerName
                    Number = order.OrderNumber,
                    Item = order.OrderItem,
                    Amount = decimal.Parse(order.OrderAmount.ToString().Replace("€", ""), CultureInfo.GetCultureInfo("en-US")),
                    CustomerName = order.CustomerName
                });
            });

            sourceOrderData.LinkTo(transIntoObject);

            //Find corresponding customer id if customer exists in Customer table
            DbSource <Customer> sourceCustomerData = new DbSource <Customer>("customer");
            LookupTransformation <Order, Customer> lookupCustomerKey = new LookupTransformation <Order, Customer>(sourceCustomerData);

            transIntoObject.LinkTo(lookupCustomerKey);

            //Split data
            Multicast <Order> multiCast = new Multicast <Order>();

            lookupCustomerKey.LinkTo(multiCast);

            //Store Order in Orders table
            DbDestination <Order> destOrderTable = new DbDestination <Order>("orders");

            multiCast.LinkTo(destOrderTable);

            //Create rating for existing customers based total of order amount
            Aggregation <Order, Rating> aggregation = new Aggregation <Order, Rating>();

            multiCast.LinkTo(aggregation);

            //Store the rating in the customer rating table
            DbDestination <Rating> destRating = new DbDestination <Rating>("customer_rating");

            aggregation.LinkTo(destRating);

            //Execute the data flow synchronously
            sourceOrderData.Execute();
            destOrderTable.Wait();
            destRating.Wait();
        }
示例#17
0
        public void reads_data_from_csv_file_with_two_columns_into_analysis_data()
        {
            var file = Path.Join(BasePath, "TwoColumns.csv");

            var csvSource = new CsvSource(file, ",");

            csvSource.Data.XUnit.ShouldBe("x");
            csvSource.Data.YUnit.ShouldBe("y");
            csvSource.Data.PointSets.ShouldBe(new [] { new Point[] { (0.1, 1), (0.2, 2), (0.3, 3) } });
示例#18
0
        public void CsvSource_CancelledLoadTest()
        {
            var tokenSource = new CancellationTokenSource();

            tokenSource.Cancel();
            var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, new RuntimeContext(tokenSource.Token, new ISchemaColumn[0]));

            var fired = source.Rows.Count();

            Assert.AreEqual(0, fired);
        }
示例#19
0
        public void CsvSource_CancelledLoadTest()
        {
            var tokenSource = new CancellationTokenSource();

            tokenSource.Cancel();
            var source = new CsvSource("./Files/BankingTransactionsWithSkippedLines.csv", ",", true, 2, new InterCommunicator(tokenSource.Token));

            var fired = source.Rows.Count();

            Assert.AreEqual(0, fired);
        }
示例#20
0
        static void Main(string[] args)
        {
            var source = new CsvSource <InputData>("Accounts_Quartal1.csv");

            source.Configuration.Delimiter = ";";

            var trans = new RowMultiplication <InputData, PivotedOutput>();

            trans.MultiplicationFunc = row =>
            {
                List <PivotedOutput> result = new List <PivotedOutput>();
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.January),
                    MonthlyValue = row.January
                });
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.February),
                    MonthlyValue = row.February
                });
                result.Add(new PivotedOutput()
                {
                    Account      = row.Account,
                    Month        = nameof(InputData.March),
                    MonthlyValue = row.March
                });
                return(result);
            };
            var dest = new CsvDestination <PivotedOutput>("AccountNumbers_Pivoted.csv");

            dest.Configuration.HasHeaderRecord = false;

            source.LinkTo(trans);
            trans.LinkTo(dest);

            Network.Execute(source);

            /* AccountNumbers_Pivoted.csv output:
             *  4711,January,10
             *  4711,February,11
             *  4711,March,12
             *  4712,January,20
             *  4712,February,21
             *  4712,March,22
             *  4713,January,30
             *  4713,February,31
             *  4713,March,32
             */
        }
示例#21
0
        public void SplitCsvSourceIn2Tables()
        {
            //Arrange
            TwoColumnsTableFixture  dest1Table = new TwoColumnsTableFixture("SplitDataDestination1");
            FourColumnsTableFixture dest2Table = new FourColumnsTableFixture("SplitDataDestination2");

            var source = new CsvSource <CSVPoco>("res/Multicast/CsvSourceToSplit.csv")
            {
                Configuration = new CsvHelper.Configuration.Configuration()
                {
                    Delimiter = ";"
                }
            };

            var multicast = new Multicast <CSVPoco>();

            var row1 = new RowTransformation <CSVPoco, Entity1>(input => {
                return(new Entity1
                {
                    Col1 = input.CSVCol1,
                    Col2 = input.CSVCol2
                });
            });
            var row2 = new RowTransformation <CSVPoco, Entity2>(input => {
                return(new Entity2
                {
                    Col2 = input.CSVCol2,
                    Col3 = input.CSVCol3,
                    Col4 = input.CSVCol4
                });
            });

            var destination1 = new DbDestination <Entity1>(Connection, "SplitDataDestination1");
            var destination2 = new DbDestination <Entity2>(Connection, "SplitDataDestination2");

            //Act
            source.LinkTo(multicast);
            multicast.LinkTo(row1);
            multicast.LinkTo(row2);

            row1.LinkTo(destination1);
            row2.LinkTo(destination2);

            source.Execute();
            destination1.Wait();
            destination2.Wait();

            //Assert
            dest1Table.AssertTestData();
            dest2Table.AssertTestData();
        }
        public void HeadersFile_ParsesWithHeaderNames()
        {
            var source = new CsvSource(_headersFile, "true", ",");
            DataTable table = source.GetData();

            Assert.Equal(_csvLines.Split(',').Length, table.Columns.Count);
            Assert.Equal(1, table.Rows.Count);

            string[] headerNames = _headers.Split(',');
            for (int idx = 0; idx < headerNames.Length; idx++)
            {
                Assert.Equal(headerNames[idx], table.Columns[idx].ColumnName);
            }
        }
        public void SimpleFlowWithDynamicObject()
        {
            //Arrange
            TwoColumnsTableFixture        dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamic");
            DbDestination <ExpandoObject> dest         = new DbDestination <ExpandoObject>("CsvSourceDynamic", SqlConnection);

            //Act
            CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void WithoutErrorLinking()
        {
            //Arrange
            MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");

            //Assert
            Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() =>
            {
                source.LinkTo(dest);
                source.Execute();
                dest.Wait();
            });
        }
示例#25
0
        public void MoreColumnsInCSV()
        {
            //Arrange
            TwoColumnsTableFixture   dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
            DbDestination <string[]> dest         = new DbDestination <string[]>(Connection, "CsvDestination2Columns");

            //Act
            CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumns.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
示例#26
0
        public void MoreColumnsInDatabase()
        {
            //Arrange
            TwoColumnsTableFixture   dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
            DbDestination <string[]> dest         = new DbDestination <string[]>(Connection, "CsvDestination2Columns");

            //Act
            CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/OneColumn.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(3, RowCountTask.Count(Connection, "CsvDestination2Columns", "Col1 IN (1,2,3)"));
        }
示例#27
0
        public void SimpleFlowWithObject()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("CsvSource2Cols");
            DbDestination <MySimpleRow> dest         = new DbDestination <MySimpleRow>("CsvSource2Cols", Connection);

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
示例#28
0
        public void IdentityAtTheEnd()
        {
            //Arrange
            FourColumnsTableFixture  dest4Columns = new FourColumnsTableFixture("CsvDestination4Columns", identityColumnIndex: 3);
            DbDestination <string[]> dest         = new DbDestination <string[]>(Connection, "CsvDestination4Columns");

            //Act
            CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumnsNoId.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest4Columns.AssertTestData();
        }
        public void MoreColumnsInSource()
        {
            //Arrange
            TwoColumnsTableFixture        dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamicColsInSource");
            DbDestination <ExpandoObject> dest         = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInSource");

            //Act
            CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/FourColumnsForDynamic.csv");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
示例#30
0
        public void SimpleLookupWithDynamicObject(IConnectionManager connection)
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SourceLookupDynamicObject");

            source2Columns.InsertTestData();
            FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture(connection, "DestinationLookupDynamicObject", -1);


            DbSource <ExpandoObject>      source = new DbSource <ExpandoObject>(connection, "SourceLookupDynamicObject");
            DbDestination <ExpandoObject> dest   = new DbDestination <ExpandoObject>(connection, "DestinationLookupDynamicObject");

            //Act
            List <ExpandoObject> lookupList = new List <ExpandoObject>();

            CsvSource <ExpandoObject> lookupSource = new CsvSource <ExpandoObject>("res/Lookup/LookupSource.csv");

            var lookup = new LookupTransformation <ExpandoObject, ExpandoObject>(
                lookupSource,
                row =>
            {
                dynamic r = row as ExpandoObject;
                r.Col3    = lookupList
                            .Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
                            .Select(lkupRow =>
                {
                    dynamic lk = lkupRow as dynamic;
                    return(lk.Column3 == string.Empty ? null : Int64.Parse(lk.Column3));
                })
                            .FirstOrDefault();
                r.Col4 = lookupList
                         .Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
                         .Select(lkupRow => { dynamic lk = lkupRow as dynamic; return(double.Parse(lk.Column4)); })
                         .FirstOrDefault();
                return(row);
            },
                lookupList
                );

            source.LinkTo(lookup);
            lookup.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest4Columns.AssertTestData();
        }
示例#31
0
        public void CSVGenericWithSkipRows_DB()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("CsvSourceSkipRows");
            DbDestination <MySimpleRow> dest         = new DbDestination <MySimpleRow>("CsvSourceSkipRows", Connection);

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsSkipRows.csv");

            source.SkipRows = 2;
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void CsvSourceNoHeader()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("CsvSourceNoHeader");
            DbDestination <MySimpleRow> dest         = new DbDestination <MySimpleRow>(Connection, "CsvSourceNoHeader");

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsNoHeader.csv");

            source.Configuration.HasHeaderRecord = false;
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void NonExistentFile_Throws()
        {
            var source = new CsvSource(@"c:\does\not\exist.csv", "false", ",");

            Assert.Throws<ArgumentException>(() => source.GetData());
        }
        public void SayHasHeaders_NoHeaders_Throws()
        {
            var source = new CsvSource(_noHeadersFile, "true", ",");

            Assert.Throws<Exception>(() => source.GetData());
        }