コード例 #1
0
        public void WithObjectErrorLinking()
        {
            //Arrange
            TwoColumnsTableFixture          dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking");
            DbDestination <MySimpleRow>     dest         = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection);
            MemoryDestination <ETLBoxError> errorDest    = new MemoryDestination <ETLBoxError>();

            //Act
            CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");

            source.LinkTo(dest);
            source.LinkErrorTo(errorDest);
            source.Execute();
            dest.Wait();
            errorDest.Wait();

            //Assert
            dest2Columns.AssertTestData();
            Assert.Collection <ETLBoxError>(errorDest.Data,
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
                                            );
        }
コード例 #2
0
        public void ExceptionInStoreKeyFunction()
        {
            //Arrange
            MemorySource <MyRow> source = new MemorySource <MyRow>();

            source.DataAsList = new List <MyRow>()
            {
                new MyRow {
                    Id = 1, ClassName = "Class1", DetailValue = 3.5
                }
            };

            //Act
            Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
                (row, aggValue) => aggValue.AggValue += row.DetailValue,
                row => row.ClassName,
                (key, agg) => throw new Exception("Test")
                );

            MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();

            source.LinkTo(agg);
            agg.LinkTo(dest);

            //Assert
            Assert.Throws <AggregateException>(() =>
            {
                source.Execute();
                dest.Wait();
            });
        }
コード例 #3
0
        public void BatchSize()
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("MemoryDestinationBatchSizeSource");

            source2Columns.InsertTestData();

            DBSource <MySimpleRow>          source = new DBSource <MySimpleRow>(SqlConnection, "MemoryDestinationBatchSizeSource");
            MemoryDestination <MySimpleRow> dest   = new MemoryDestination <MySimpleRow>()
            {
                BatchSize = 2
            };

            //Act
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection(dest.Data,
                              d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
                              d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
                              d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
                              );
        }
コード例 #4
0
        public void ExceptionInAggregationFunction()
        {
            //Arrange
            MemorySource <MyRow> source = new MemorySource <MyRow>();

            source.DataAsList = new List <MyRow>()
            {
                new MyRow {
                    Id = 1, DetailValue = 3.5
                },
            };

            //Act
            Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
                (row, aggRow) => throw new Exception("Test")
                );

            MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();

            //Assert
            source.LinkTo(agg);
            agg.LinkTo(dest);

            Assert.Throws <AggregateException>(() =>
            {
                source.Execute();
                dest.Wait();
            });
        }
コード例 #5
0
        public void UsingRowTransformation()
        {
            Prepare();
            var orderSource = new MemorySource <Order>();

            orderSource.DataAsList.Add(new Order()
            {
                OrderNumber = 815, CustomerName = "John"
            });
            orderSource.DataAsList.Add(new Order()
            {
                OrderNumber = 4711, CustomerName = "Jim"
            });

            var rowTrans = new RowTransformation <Order>(
                row => {
                int?id = SqlTask.ExecuteScalar <int>(SqlConnection,
                                                     sql: $"SELECT Id FROM CustomerTable WHERE Name='{row.CustomerName}'");
                row.CustomerId = id;
                return(row);
            });


            /* Delete below here */

            var dest = new MemoryDestination <Order>();

            orderSource.LinkTo(rowTrans).LinkTo(dest);
            orderSource.Execute();
        }
コード例 #6
0
        public void UsingLookupWithAttributes()
        {
            Prepare();
            var orderSource = new MemorySource <Order>();

            orderSource.DataAsList.Add(new Order()
            {
                OrderNumber = 815, CustomerName = "John"
            });
            orderSource.DataAsList.Add(new Order()
            {
                OrderNumber = 4711, CustomerName = "Jim"
            });

            var lookupSource = new DbSource <CustomerWithAttr>(SqlConnection, "CustomerTable");

            var lookup = new LookupTransformation <Order, CustomerWithAttr>();

            lookup.Source = lookupSource;

            var dest = new MemoryDestination <Order>();

            orderSource.LinkTo(lookup).LinkTo(dest);
            Network.Execute(orderSource);

            foreach (var row in dest.Data)
            {
                Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
            }

            //Output
            //Order:815 Name:John Id:1
            //Order:4711 Name:Jim Id:2
        }
コード例 #7
0
        public void AggregateSumWithNullable()
        {
            //Arrange
            List <MyInputRow> sourceData = new List <MyInputRow>()
            {
                new MyInputRow {
                    Id = 1, DetailValue = 3.5
                },
                new MyInputRow {
                    Id = 2, DetailValue = 4.5
                },
                new MyInputRow {
                    Id = 3, DetailValue = 2.0
                },
                new MyInputRow {
                    Id = 4, DetailValue = null
                },
            };
            MemoryDestination <MySumRowNullable> dest = CreateFlow <MySumRowNullable>(sourceData);

            //Assert
            Assert.Collection(dest.Data,
                              ar => Assert.True(ar.AggValue == 10)
                              );
        }
コード例 #8
0
        public void WithPredicate()
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationSource");

            source2Columns.InsertTestData();

            DbSource <MySimpleRow>       source      = new DbSource <MySimpleRow>(SqlConnection, "RowDuplicationSource");
            RowDuplication <MySimpleRow> duplication = new RowDuplication <MySimpleRow>(
                row => row.Col1 == 1 || row.Col2 == "Test3"
                );
            MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();

            //Act
            source.LinkTo(duplication);
            duplication.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection(dest.Data,
                              d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
                              d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
                              d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
                              d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3"),
                              d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
                              );
        }
コード例 #9
0
        public void WithPredicate()
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationSource");

            source2Columns.InsertTestData();

            DbSource       source      = new DbSource(SqlConnection, "RowDuplicationSource");
            RowDuplication duplication = new RowDuplication(
                row =>
            {
                dynamic r = row as dynamic;
                return(r.Col1 == 1 || r.Col2 == "Test3");
            });
            MemoryDestination dest = new MemoryDestination();

            //Act
            source.LinkTo(duplication);
            duplication.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection(dest.Data,
                              row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
                              row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
                              row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 2 && d.Col2 == "Test2"); },
                              row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); },
                              row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); }
                              );
        }
コード例 #10
0
        public void NoLookupSource()
        {
            //Arrange
            MemorySource <MyDataRow> source = new MemorySource <MyDataRow>();

            source.DataAsList.Add(new MyDataRow()
            {
                Col1 = 1, Col2 = "Test1"
            });

            //Act
            var lookup = new LookupTransformation <MyDataRow, MyLookupRow>();
            MemoryDestination <MyDataRow> dest = new MemoryDestination <MyDataRow>();

            //Assert
            Assert.Throws <ETLBoxException>(() =>
            {
                try
                {
                    source.LinkTo(lookup);
                    lookup.LinkTo(dest);
                    source.Execute();
                    dest.Wait();
                }
                catch (AggregateException e) { throw e.InnerException; }
            });
            //Assert
        }
コード例 #11
0
        public void ValidateSchemaForArray()
        {
            //Arrange
            var source = new MemorySource <string[]>();

            source.DataAsList.Add(new string[] { _validXml });
            source.DataAsList.Add(new string[] { _invalidXml });
            source.DataAsList.Add(new string[] { _validXml });

            MemoryDestination <string[]>    dest  = new MemoryDestination <string[]>();
            MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();

            //Act
            XmlSchemaValidation <string[]> schemaValidation = new XmlSchemaValidation <string[]>();

            schemaValidation.XmlSelector = row => row[0];
            schemaValidation.XmlSchema   = xsdMarkup;

            source.LinkTo(schemaValidation);
            schemaValidation.LinkTo(dest);
            schemaValidation.LinkErrorTo(error);
            source.Execute();
            dest.Wait();
            error.Wait();

            //Assert
            Assert.True(dest.Data.Count == 2);
            Assert.True(error.Data.Count == 1);
        }
コード例 #12
0
        public void ThrowExceptionWithoutHandling()
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowMultiplicationSource");

            source2Columns.InsertTestData();

            DbSource <MySimpleRow>          source         = new DbSource <MySimpleRow>(SqlConnection, "RowMultiplicationSource");
            RowMultiplication <MySimpleRow> multiplication = new RowMultiplication <MySimpleRow>(
                row =>
            {
                List <MySimpleRow> result = new List <MySimpleRow>();
                result.Add(row);
                if (row.Col1 == 2)
                {
                    throw new Exception("Error in Flow!");
                }
                return(result);
            });
            MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();

            //Act & Assert
            source.LinkTo(multiplication);
            multiplication.LinkTo(dest);

            Assert.Throws <AggregateException>(() =>
            {
                source.Execute();
                dest.Wait();
            });
        }
コード例 #13
0
ファイル: TextSourceTests.cs プロジェクト: Sunny-365/etlbox
        public void ReadingIntoDynamicObject()
        {
            //Arrange
            var dest = new MemoryDestination();

            //Act
            TextSource source = new TextSource();

            source.Uri = "res/TextSource/Test.txt";
            source.WriteLineIntoObject = (line, dynob) =>
            {
                dynamic o = dynob as ExpandoObject;
                o.Text = line;
            };
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.True(dest.Data.Count == 3);
            Assert.Collection <ExpandoObject>(dest.Data,
                                              row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 1", r.Text); },
                                              row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 2", r.Text); },
                                              row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 3", r.Text); }
                                              );
        }
コード例 #14
0
        public void CompareFlowWithBulkInsert(int numberOfRows, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);

            var sourceNonGeneric = new CSVSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destNonGeneric   = new MemoryDestination();
            var sourceGeneric    = new CSVSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destGeneric      = new MemoryDestination <CSVData>();
            var sourceDynamic    = new CSVSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destDynamic      = new MemoryDestination <ExpandoObject>();


            //Act
            var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
            var teGeneric    = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
            var teDynamic    = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);

            //Assert
            Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
            Assert.Equal(numberOfRows, destGeneric.Data.Count);
            Assert.Equal(numberOfRows, destDynamic.Data.Count);
            Assert.True(new [] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
                        new [] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
        }
コード例 #15
0
        public void GroupingUsingStringArray()
        {
            //Arrange
            MemorySource <string[]> source = new MemorySource <string[]>();

            source.DataAsList.Add(new string[] { "Class1", "3.5" });
            source.DataAsList.Add(new string[] { "Class1", "6.5" });
            source.DataAsList.Add(new string[] { "Class2", "10" });

            Aggregation <string[], MyAggRow> agg = new Aggregation <string[], MyAggRow>(
                (row, aggValue) => aggValue.AggValue += Convert.ToDouble(row[1]),
                row => row[0],
                (key, agg) => agg.GroupName = (string)key
                );

            MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();

            //Act
            source.LinkTo(agg);
            agg.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <MyAggRow>(dest.Data,
                                         ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
                                         ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class2")
                                         );
        }
コード例 #16
0
        public void ErrorInSql()
        {
            //Arrange
            DbSource source = new DbSource(SqlConnection)
            {
                Sql = "SELECT XYZ FROM ABC"
            };
            MemoryDestination dest = new MemoryDestination();

            source.LinkTo(dest);
            //Act & Assert
            Assert.Throws <SqlException>(() =>
            {
                try
                {
                    Task s = source.ExecuteAsync();
                    Task c = dest.Completion;
                    Task.WaitAll(c, s);
                }
                catch (AggregateException e)
                {
                    throw e.InnerException;
                }
            });
        }
コード例 #17
0
        public void DataIsInList()
        {
            //Arrange
            TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationStringArraySource");

            source2Columns.InsertTestData();

            DbSource <string[]>          source      = new DbSource <string[]>(SqlConnection, "RowDuplicationStringArraySource");
            RowDuplication <string[]>    duplication = new RowDuplication <string[]>();
            MemoryDestination <string[]> dest        = new MemoryDestination <string[]>();

            //Act
            source.LinkTo(duplication);
            duplication.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection(dest.Data,
                              d => Assert.True(d[0] == "1" && d[1] == "Test1"),
                              d => Assert.True(d[0] == "1" && d[1] == "Test1"),
                              d => Assert.True(d[0] == "2" && d[1] == "Test2"),
                              d => Assert.True(d[0] == "2" && d[1] == "Test2"),
                              d => Assert.True(d[0] == "3" && d[1] == "Test3"),
                              d => Assert.True(d[0] == "3" && d[1] == "Test3")
                              );
        }
コード例 #18
0
        public void RedirectErrorWithObject(IConnectionManager connection)
        {
            if (connection.GetType() == typeof(SQLiteConnectionManager))
            {
                Task.Delay(100).Wait(); //Database was locked and needs to recover after exception
            }
            //Arrange
            CreateSourceTable(connection, "DbSourceErrorLinking");
            TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "DbDestinationErrorLinking");

            //Act
            DbSource <MySimpleRow>          source    = new DbSource <MySimpleRow>(connection, "DbSourceErrorLinking");
            DbDestination <MySimpleRow>     dest      = new DbDestination <MySimpleRow>(connection, "DbDestinationErrorLinking");
            MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();

            source.LinkTo(dest);
            source.LinkErrorTo(errorDest);
            source.Execute();
            dest.Wait();
            errorDest.Wait();

            //Assert
            dest2Columns.AssertTestData();
            Assert.Collection <ETLBoxError>(errorDest.Data,
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
                                            );
        }
コード例 #19
0
        private static void RunExceptionFlowWithType <T>()
        {
            //Arrange
            MemorySource <InputDataRow> source = new MemorySource <InputDataRow>();

            source.DataAsList.Add(new InputDataRow()
            {
                LookupId = 1
            });
            MemorySource <T> lookupSource = new MemorySource <T>();
            var lookup = new LookupTransformation <InputDataRow, T>(lookupSource);
            MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>();

            source.LinkTo(lookup);
            lookup.LinkTo(dest);

            //Act && Assert
            Assert.Throws <ETLBoxException>(() =>
            {
                try
                {
                    source.Execute();
                    dest.Wait();
                }
                catch (AggregateException e)
                {
                    throw e.InnerException;
                }
            });
        }
コード例 #20
0
        public void AggregateSimple()
        {
            //Arrange
            MemorySource <MyRow> source = new MemorySource <MyRow>();

            source.DataAsList = new List <MyRow>()
            {
                new MyRow {
                    Id = 1, DetailValue = 3.5
                },
                new MyRow {
                    Id = 2, DetailValue = 4.5
                },
                new MyRow {
                    Id = 3, DetailValue = 2.0
                },
            };

            Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
                (row, aggRow) => aggRow.AggValue += row.DetailValue
                );

            MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();

            //Act
            source.LinkTo(agg);
            agg.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <MyAggRow>(dest.Data,
                                         ar => Assert.True(ar.AggValue == 10)
                                         );
        }
コード例 #21
0
        public void DynamicObjectJoin()
        {
            //Arrange
            TwoColumnsTableFixture table1 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource1");

            table1.InsertTestData();
            TwoColumnsTableFixture table2 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource2");

            table2.InsertTestData();
            DbSource <ExpandoObject> source1 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource1");
            DbSource <ExpandoObject> source2 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource2");
            MemoryDestination        dest    = new MemoryDestination();

            CrossJoin crossJoin = new CrossJoin(
                (data1, data2) =>
            {
                dynamic d1  = data1 as dynamic;
                dynamic d2  = data1 as dynamic;
                dynamic res = new ExpandoObject();
                res.Val     = d1.Col1 + d2.Col2;
                return(res);
            }
                );

            //Act
            source1.LinkTo(crossJoin.InMemoryTarget);
            source2.LinkTo(crossJoin.PassingTarget);
            crossJoin.LinkTo(dest);
            source1.Execute();
            source2.Execute();
            dest.Wait();

            //Assert
            Assert.Equal(9, dest.Data.Count);
        }
コード例 #22
0
        public void ValidateSchemaForDynamicObject()
        {
            //Arrange
            var     source = new MemorySource();
            dynamic n1     = new ExpandoObject(); n1.Xml = _validXml; source.DataAsList.Add(n1);
            dynamic n2     = new ExpandoObject(); n2.Xml = _validXml; source.DataAsList.Add(n2);
            dynamic n3     = new ExpandoObject(); n3.Xml = _invalidXml; source.DataAsList.Add(n3);

            MemoryDestination dest = new MemoryDestination();
            MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();

            //Act
            XmlSchemaValidation schemaValidation = new XmlSchemaValidation();

            schemaValidation.XmlSelector = row =>
            {
                dynamic r = row as ExpandoObject;
                return(r.Xml);
            };
            schemaValidation.XmlSchema = xsdMarkup;

            source.LinkTo(schemaValidation);
            schemaValidation.LinkTo(dest);
            schemaValidation.LinkErrorTo(error);
            source.Execute();
            dest.Wait();
            error.Wait();

            //Assert
            Assert.True(dest.Data.Count == 2);
            Assert.True(error.Data.Count == 1);
        }
コード例 #23
0
        public void GroupingAndKeepingKey()
        {
            //Arrange
            MemorySource <MyRow> source = new MemorySource <MyRow>();

            source.Data = new List <MyRow>()
            {
                new MyRow {
                    Id = 1, ClassName = "Class1", DetailValue = 3.5
                },
                new MyRow {
                    Id = 2, ClassName = "Class1", DetailValue = 6.5
                },
                new MyRow {
                    Id = 3, ClassName = "Class2", DetailValue = 1.2
                },
                new MyRow {
                    Id = 4, ClassName = "Class2", DetailValue = 2.3
                },
                new MyRow {
                    Id = 5, ClassName = "Class2", DetailValue = 16.5
                },
                new MyRow {
                    Id = 6, ClassName = "Class3", DetailValue = 30.0
                },
                new MyRow {
                    Id = 6, ClassName = null, DetailValue = 14.5
                },
                new MyRow {
                    Id = 6, ClassName = null, DetailValue = 15.5
                },
            };

            Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
                (row, aggValue) => aggValue.AggValue += row.DetailValue,
                row => row.ClassName,
                (key, agg) => agg.GroupName = (string)key
                );

            MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();

            //Act
            source.LinkTo(agg);
            agg.LinkTo(dest);
            source.Execute();
            dest.Wait();


            //Assert
            Assert.Collection <MyAggRow>(dest.Data,
                                         ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
                                         ar => Assert.True(ar.AggValue == 20 && ar.GroupName == "Class2"),
                                         ar => Assert.True(ar.AggValue == 30 && ar.GroupName == "Class3"),
                                         ar => Assert.True(ar.AggValue == 30 && ar.GroupName == string.Empty)
                                         );
        }
コード例 #24
0
ファイル: CrossJoin.cs プロジェクト: Sunny-365/etlbox
 protected override void InitBufferObjects()
 {
     InMemoryTarget = new MemoryDestination <TInput1>(this);
     PassingTarget  = new CustomDestination <TInput2>(this, CrossJoinData);
     if (MaxBufferSize > 0)
     {
         PassingTarget.MaxBufferSize = this.MaxBufferSize;
     }
     PassingTarget.OnCompletion = () => Buffer.Complete();
 }
コード例 #25
0
        public void WaitForLongRunningProcedure()
        {
            //Arrange
            CreateProcedureTask.CreateOrAlter(SqlConnection, new ProcedureDefinition()
            {
                Name       = "longrunning",
                Definition = @"
WAITFOR DELAY '00:00:01';
SELECT @input + 10;",
                Parameter  = new List <ProcedureParameter>()
                {
                    new ProcedureParameter()
                    {
                        Name = "input", DataType = "INT"
                    }
                    // new ProcedureParameter() { Name ="result", Out = true, DataType="INT"}
                }
            });
            var source = new MemorySource <MySimpleRow>();

            source.DataAsList.Add(new MySimpleRow()
            {
                Col1 = 1, Col2 = "Test1"
            });
            source.DataAsList.Add(new MySimpleRow()
            {
                Col1 = 2, Col2 = "Test2"
            });
            source.DataAsList.Add(new MySimpleRow()
            {
                Col1 = 3, Col2 = "Test3"
            });
            var dest  = new MemoryDestination <MySimpleRow>();
            var trans = new RowTransformation <MySimpleRow>(
                row =>
            {
                row.Col1 = SqlTask.ExecuteScalar <int>(SqlConnection, "Read from procedure",
                                                       $"EXEC longrunning @input = {row.Col1}") ?? 0;
                return(row);
            }
                );

            //Act
            source.LinkTo(trans);
            trans.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <MySimpleRow>(dest.Data,
                                            row => Assert.Equal(11, row.Col1),
                                            row => Assert.Equal(12, row.Col1),
                                            row => Assert.Equal(13, row.Col1)
                                            );
        }
コード例 #26
0
        public void OneMatchOneRetrieveColumn()
        {
            //Arrange
            MemorySource <InputDataRow> source = new MemorySource <InputDataRow>();

            source.DataAsList.Add(new InputDataRow()
            {
                LookupId = 1
            });
            source.DataAsList.Add(new InputDataRow()
            {
                LookupId = 2
            });
            source.DataAsList.Add(new InputDataRow()
            {
                LookupId = 4
            });
            source.DataAsList.Add(new InputDataRow()
            {
                LookupId = 3
            });
            MemorySource <LookupData> lookupSource = new MemorySource <LookupData>();

            lookupSource.DataAsList.Add(new LookupData()
            {
                Id = 1, Value = "Test1"
            });
            lookupSource.DataAsList.Add(new LookupData()
            {
                Id = 2, Value = "Test2"
            });
            lookupSource.DataAsList.Add(new LookupData()
            {
                Id = 3, Value = "Test3"
            });

            var lookup = new LookupTransformation <InputDataRow, LookupData>();

            lookup.Source = lookupSource;
            MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>();

            source.LinkTo(lookup);
            lookup.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <InputDataRow>(dest.Data,
                                             r => Assert.True(r.LookupId == 1 && r.LookupValue == "Test1"),
                                             r => Assert.True(r.LookupId == 2 && r.LookupValue == "Test2"),
                                             r => Assert.True(r.LookupId == 4 && r.LookupValue == null),
                                             r => Assert.True(r.LookupId == 3 && r.LookupValue == "Test3")
                                             );
        }
コード例 #27
0
        public void JsonFromWebService()
        {
            JsonSource        source = new JsonSource("res/JsonSource/EmptyObject.json", ResourceType.File);
            MemoryDestination dest   = new MemoryDestination();

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            Assert.True(dest.Data.Count == 0);
        }
コード例 #28
0
        public void RedirectBatch(IConnectionManager connection)
        {
            //Arrange
            TwoColumnsTableFixture     d2c    = new TwoColumnsTableFixture(connection, "DestLinkError");
            MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();

            source.Data = new List <MySimpleRow>()
            {
                new MySimpleRow()
                {
                    Col1 = null, Col2 = "ErrorRecord"
                },
                new MySimpleRow()
                {
                    Col1 = "X2", Col2 = "ErrorRecord"
                },
                new MySimpleRow()
                {
                    Col1 = "1", Col2 = "Test1"
                },
                new MySimpleRow()
                {
                    Col1 = "2", Col2 = "Test2"
                },
                new MySimpleRow()
                {
                    Col1 = "3", Col2 = "Test3 - good, but in error batch"
                },
                new MySimpleRow()
                {
                    Col1 = null, Col2 = "ErrorRecord"
                },
                new MySimpleRow()
                {
                    Col1 = "3", Col2 = "Test3"
                },
            };
            DBDestination <MySimpleRow>     dest      = new DBDestination <MySimpleRow>(connection, "DestLinkError", batchSize: 2);
            MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();

            //Act
            source.LinkTo(dest);
            dest.LinkErrorTo(errorDest);
            source.Execute();
            dest.Wait();
            errorDest.Wait();

            //Assert
            d2c.AssertTestData();
            Assert.Collection <ETLBoxError>(errorDest.Data,
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
                                            d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
                                            );
        }
コード例 #29
0
        public void MultipleMatchAndRetrieveColumns()
        {
            //Arrange
            MemorySource <InputDataMultiple> source = new MemorySource <InputDataMultiple>();

            source.DataAsList.Add(new InputDataMultiple()
            {
                LookupId1 = 1, LookupId2 = "T1"
            });
            source.DataAsList.Add(new InputDataMultiple()
            {
                LookupId1 = 2, LookupId2 = "TX"
            });
            source.DataAsList.Add(new InputDataMultiple()
            {
                LookupId1 = 4, LookupId2 = "T2"
            });
            source.DataAsList.Add(new InputDataMultiple()
            {
                LookupId1 = 3, LookupId2 = "T3"
            });
            MemorySource <LookupDataMultiple> lookupSource = new MemorySource <LookupDataMultiple>();

            lookupSource.DataAsList.Add(new LookupDataMultiple()
            {
                Id1 = 1, Id2 = "T1", Value1 = "Test1", Value2 = 100
            });
            lookupSource.DataAsList.Add(new LookupDataMultiple()
            {
                Id1 = 2, Value1 = "Test2", Value2 = 200
            });
            lookupSource.DataAsList.Add(new LookupDataMultiple()
            {
                Id1 = 3, Id2 = "T3", Value2 = 300
            });

            var lookup = new LookupTransformation <InputDataMultiple, LookupDataMultiple>();

            lookup.Source = lookupSource;
            MemoryDestination <InputDataMultiple> dest = new MemoryDestination <InputDataMultiple>();

            source.LinkTo(lookup);
            lookup.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <InputDataMultiple>(dest.Data,
                                                  r => Assert.True(r.LookupId1 == 1 && r.LookupValue1 == "Test1" && r.LookupValue2 == 100),
                                                  r => Assert.True(r.LookupId1 == 2 && r.LookupValue1 == null),
                                                  r => Assert.True(r.LookupId1 == 4 && r.LookupValue1 == null),
                                                  r => Assert.True(r.LookupId1 == 3 && r.LookupValue1 == null && r.LookupValue2 == 300)
                                                  );
        }
コード例 #30
0
        public void ReadEmptyArray()
        {
            JsonSource        source = new JsonSource("res/JsonSource/EmptyArray.json", ResourceType.File);
            MemoryDestination dest   = new MemoryDestination();

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            Assert.True(dest.Data.Count == 0);
        }