Exemplo n.º 1
0
        public void CSVIntoMemDest(int numberOfRows, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);

            var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destNonGeneric   = new MemoryDestination();
            var sourceGeneric    = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destGeneric      = new MemoryDestination <CSVData>();
            var sourceDynamic    = new CsvSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destDynamic      = new MemoryDestination <ExpandoObject>();

            sourceNonGeneric.ReleaseGCPressureRowCount = 500;
            sourceGeneric.ReleaseGCPressureRowCount    = 500;
            sourceDynamic.ReleaseGCPressureRowCount    = 500;
            //Act
            var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
            var teGeneric    = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
            var teDynamic    = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);

            //Assert
            Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
            Assert.Equal(numberOfRows, destGeneric.Data.Count);
            Assert.Equal(numberOfRows, destDynamic.Data.Count);
            Assert.True(new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
                        new[] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
        }
Exemplo n.º 2
0
        public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
            ReCreateDestinationTable(connection, "CsvDestinationWithTransformation");

            var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var trans         = new RowTransformation <ExpandoObject, CSVData>(
                row =>
            {
                dynamic r = row as ExpandoObject;
                return(new CSVData()
                {
                    Col1 = r.Col1,
                    Col2 = r.Col2,
                    Col3 = r.Col3,
                    Col4 = r.Col4
                });
            });
            var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize);

            sourceExpando.LinkTo(trans);
            trans.LinkTo(destGeneric);

            //Act
            long memAfter   = 0;
            long memBefore  = 0;
            bool startCheck = true;
            int  count      = 1;

            destGeneric.AfterBatchWrite = data =>
            {
                if (count++ % 50 == 0)
                {
                    using Process proc = Process.GetCurrentProcess();
                    memAfter           = proc.WorkingSet64;
                    if (startCheck)
                    {
                        memBefore  = memAfter;
                        startCheck = false;
                    }
                    Assert.True(memAfter < (memBefore + (memBefore * deviation)));
                }
            };

            var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
                                                                   () =>
            {
                sourceExpando.Execute();
                destGeneric.Wait();
            }
                                                                   );

            output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation).");

            //Assert
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation"));
            //10.000.000 rows, batch size 10.000: ~8 min
            //10.000.000 rows, batch size  1.000: ~10 min 10 sec
        }
Exemplo n.º 3
0
        public void CompareGenericAndDynamic(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
            ReCreateDestinationTable(connection, "CsvDestinationNonGenericETLBox");
            ReCreateDestinationTable(connection, "CsvDestinationBulkInsert");
            ReCreateDestinationTable(connection, "CsvDestinationGenericETLBox");

            var sourceNonGeneric = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destNonGeneric   = new DbDestination(connection, "CsvDestinationNonGenericETLBox", batchSize);
            var sourceGeneric    = new CsvSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var destGeneric      = new DbDestination <CSVData>(connection, "CsvDestinationGenericETLBox", batchSize);

            //Act
            var timeElapsedETLBoxNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
            var timeElapsedETLBoxGeneric    = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);

            //Assert
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationNonGenericETLBox"));
            Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationGenericETLBox"));
            Assert.True(Math.Abs(timeElapsedETLBoxGeneric.TotalMilliseconds - timeElapsedETLBoxNonGeneric.TotalMilliseconds) <
                        Math.Min(timeElapsedETLBoxGeneric.TotalMilliseconds, timeElapsedETLBoxNonGeneric.TotalMilliseconds) * deviation);
        }
Exemplo n.º 4
0
        public void WriteParallelWhileTransactionOpen(IConnectionManager connection, int numberOfRows)
        {
            //Arrange
            BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
            ReCreateDestinationTable(connection, "TransactionDestination1");
            ReCreateDestinationTable(connection, "TransactionDestination2");

            var source = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
            var dest1  = new DbDestination(connection, "TransactionDestination1");
            var dest2  = new DbDestination(connection, "TransactionDestination2");
            var multi  = new Multicast();

            //Act & Assert
            Assert.ThrowsAny <Exception>(() =>
            {
                connection.BeginTransaction();
                source.LinkTo(multi);
                multi.LinkTo(dest1);
                multi.LinkTo(dest2);
                source.Execute();
                dest1.Wait();
                dest2.Wait();
            });
        }