Exemple #1
0
        public override void CommitTableDataCompleted(DiscoveredTable resultingTable)
        {
            //if user has an explicit type to use for the column (probably a good idea to have all extraction idetntifiers of the same data type
            var col = resultingTable.DiscoverColumn(_identifierColumn.ColumnName);

            CreateIndex(resultingTable, col, Configuration.ToString());
        }
        public override void DropTable(DbConnection connection, DiscoveredTable tableToDrop)
        {
            SqlCommand cmd;

            switch (tableToDrop.TableType)
            {
            case TableType.View:
                if (connection.Database != tableToDrop.Database.GetRuntimeName())
                {
                    connection.ChangeDatabase(tableToDrop.GetRuntimeName());
                }

                if (!connection.Database.ToLower().Equals(tableToDrop.Database.GetRuntimeName().ToLower()))
                {
                    throw new NotSupportedException("Cannot drop view " + tableToDrop + " because it exists in database " + tableToDrop.Database.GetRuntimeName() + " while the current current database connection is pointed at database:" + connection.Database + " (use .ChangeDatabase on the connection first) - SQL Server does not support cross database view dropping");
                }

                cmd = new SqlCommand("DROP VIEW " + tableToDrop.GetRuntimeName(), (SqlConnection)connection);
                break;

            case TableType.Table:
                cmd = new SqlCommand("DROP TABLE " + tableToDrop.GetFullyQualifiedName(), (SqlConnection)connection);
                break;

            case TableType.TableValuedFunction:
                DropFunction(connection, (DiscoveredTableValuedFunction)tableToDrop);
                return;

            default:
                throw new ArgumentOutOfRangeException();
            }

            using (cmd)
                cmd.ExecuteNonQuery();
        }
Exemple #3
0
        private static HashSet <string> FetchTable(ColumnInfo columnInfo)
        {
            var logger = LogManager.GetCurrentClassLogger();
            HashSet <string> toReturn = new HashSet <string>(StringComparer.CurrentCultureIgnoreCase);

            var qb = new QueryBuilder(limitationSQL: null, hashingAlgorithm: null);

            qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), columnInfo));

            string sql = qb.SQL;

            logger.Info("Running PatientID fetch SQL:" + sql);

            DiscoveredTable server = columnInfo.TableInfo.Discover(DataAccessContext.DataExport);

            using (DbConnection con = server.Database.Server.GetConnection())
            {
                con.Open();
                DbCommand    cmd    = server.GetCommand(sql, con);
                DbDataReader reader = cmd.ExecuteReader();

                while (reader.Read())
                {
                    toReturn.Add(reader[0].ToString());
                }
            }

            logger.Info($"Found {toReturn.Count} patients in the reject list");

            return(toReturn);
        }
        public override int ExecuteInsertReturningIdentity(DiscoveredTable discoveredTable, DbCommand cmd, IManagedTransaction transaction = null)
        {
            var autoIncrement = discoveredTable.DiscoverColumns(transaction).SingleOrDefault(c => c.IsAutoIncrement);

            if (autoIncrement == null)
            {
                return(Convert.ToInt32(cmd.ExecuteScalar()));
            }

            var p = discoveredTable.Database.Server.Helper.GetParameter("identityOut");

            p.Direction = ParameterDirection.Output;
            p.DbType    = DbType.Int32;

            cmd.Parameters.Add(p);

            cmd.CommandText += " RETURNING " + autoIncrement + " INTO :identityOut;";

            cmd.CommandText = "BEGIN " + Environment.NewLine + cmd.CommandText + Environment.NewLine + "COMMIT;" + Environment.NewLine + "END;";

            cmd.ExecuteNonQuery();


            return(Convert.ToInt32(p.Value));
        }
        private void ForwardEngineer(DiscoveredTable targetTableName)
        {
            var extractionPicker = new ConfigureCatalogueExtractabilityUI(Activator, new TableInfoImporter(Activator.RepositoryLocator.CatalogueRepository, targetTableName), "File '" + _selectedFile.FullName + "'", _projectSpecific)
            {
                TargetFolder = TargetFolder,
                TableCreated = targetTableName
            };

            extractionPicker.ShowDialog();

            var catalogue = extractionPicker.CatalogueCreatedIfAny;

            if (catalogue is DatabaseEntity de)
            {
                Activator.RefreshBus.Publish(this, new RefreshObjectEventArgs(de));

                MessageBox.Show("Successfully imported new Dataset '" + catalogue + "'." +
                                "\r\n" +
                                "The edit functionality will now open.");

                Activator.WindowArranger.SetupEditAnything(this, catalogue);
            }
            if (cbAutoClose.Checked)
            {
                this.Close();
            }
            else
            {
                MessageBox.Show("Creation completed successfully, close the Form when you are finished reviewing the output");
            }
        }
Exemple #6
0
        protected override string BuildUpdateImpl(DiscoveredTable table1, DiscoveredTable table2, List <CustomLine> lines)
        {
            // This implementation is based on:
            // https://stackoverflow.com/a/32748797/4824531

            /*MERGE INTO table1 t1
             * USING
             * (
             * -- For more complicated queries you can use WITH clause here
             * SELECT * FROM table2
             * )t2
             * ON(t1.id = t2.id)
             * WHEN MATCHED THEN UPDATE SET
             * t1.name = t2.name,
             * t1.desc = t2.desc;*/


            return(string.Format(
                       @"MERGE INTO {1} t1
USING
( 
    SELECT * FROM {2}
)t2
on ({3})
WHEN MATCHED THEN UPDATE SET
    {0}
WHERE
{4}",
                       string.Join(", " + Environment.NewLine, lines.Where(l => l.LocationToInsert == QueryComponent.SET).Select(c => c.Text)),
                       table1.GetFullyQualifiedName(),
                       table2.GetFullyQualifiedName(),
                       string.Join(" AND ", lines.Where(l => l.LocationToInsert == QueryComponent.JoinInfoJoin).Select(c => c.Text)),
                       string.Join(" AND ", lines.Where(l => l.LocationToInsert == QueryComponent.WHERE).Select(c => c.Text))));
        }
        public override void MakeDistinct(DiscoveredTable discoveredTable, int timeoutInSeconds)
        {
            string sql =
                @"DELETE f
            FROM (
            SELECT	ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {0}) AS RowNum
            FROM {1}
            
            ) as f
            where RowNum > 1";

            string columnList = string.Join(",", discoveredTable.DiscoverColumns().Select(c => c.GetRuntimeName()));

            string sqlToExecute = string.Format(sql, columnList, discoveredTable.GetFullyQualifiedName());

            var server = discoveredTable.Database.Server;

            using (var con = server.GetConnection())
            {
                con.Open();
                var cmd = server.GetCommand(sqlToExecute, con);
                cmd.CommandTimeout = timeoutInSeconds;
                cmd.ExecuteNonQuery();
            }
        }
Exemple #8
0
        protected override string BuildUpdateImpl(DiscoveredTable table1, DiscoveredTable table2, List <CustomLine> lines)
        {
            //https://stackoverflow.com/a/7869611
            string joinSql = string.Join(" AND ",
                                         lines.Where(l => l.LocationToInsert == QueryComponent.JoinInfoJoin).Select(c => c.Text));

            string whereSql = string.Join(" AND ",
                                          lines.Where(l => l.LocationToInsert == QueryComponent.WHERE).Select(c => c.Text));

            return(string.Format(
                       @"UPDATE {1} AS t1
SET 
    {0}
FROM
 {2} AS t2 
WHERE
{3}
{4}
{5}
",

                       string.Join(", " + Environment.NewLine, lines.Where(l => l.LocationToInsert == QueryComponent.SET)
                                   .Select(c =>
                                           //seems like you cant specify the table alias in the SET section of the query
                                           c.Text.Replace("t1.", ""))),
                       table1.GetFullyQualifiedName(),
                       table2.GetFullyQualifiedName(),
                       joinSql,
                       !string.IsNullOrWhiteSpace(whereSql) ? "AND" :"",
                       !string.IsNullOrWhiteSpace(whereSql) ? "(" + whereSql + ")":""
                       ));
        }
        private TableInfo Import(DiscoveredTable tbl, LoadMetadata lmd, LogManager logManager)
        {
            logManager.CreateNewLoggingTaskIfNotExists(lmd.Name);

            //import TableInfos
            var       importer = new TableInfoImporter(CatalogueRepository, tbl);
            TableInfo ti;

            ColumnInfo[] cis;
            importer.DoImport(out ti, out cis);

            //create Catalogue
            var forwardEngineer = new ForwardEngineerCatalogue(ti, cis, true);

            Catalogue cata;

            CatalogueItem[]         cataItems;
            ExtractionInformation[] eis;
            forwardEngineer.ExecuteForwardEngineering(out cata, out cataItems, out eis);

            //make the catalogue use the load configuration
            cata.LoadMetadata_ID = lmd.ID;
            cata.LoggingDataTask = lmd.Name;
            Assert.IsNotNull(cata.LiveLoggingServer_ID); //catalogue should have one of these because of system defaults
            cata.SaveToDatabase();

            return(ti);
        }
Exemple #10
0
        public void SubstituteANOIdentifiers_PreviewWithoutPush()
        {
            var anoTable = GetANOTable();

            anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
            anoTable.NumberOfIntegersToUseInAnonymousRepresentation   = 10;

            DiscoveredTable ANOtable = ANOStore_Database.ExpectTable(anoTable.TableName);

            //should not exist yet
            Assert.False(ANOtable.Exists());

            DataTable dt = new DataTable();

            dt.Columns.Add("CHI");
            dt.Columns.Add("ANOCHI");
            dt.Rows.Add("0101010101", DBNull.Value);
            ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);

            Assert.IsTrue(_anochiPattern.IsMatch((string)dt.Rows[0][1]));//should be 10 digits and then _A

            //still not exist yet
            Assert.False(ANOtable.Exists());

            anoTable.DeleteInDatabase();
        }
        private string[] ListPrimaryKeys(IManagedConnection con, DiscoveredTable table)
        {
            List <string> toReturn = new List <string>();

            string query = String.Format(@"SELECT i.name AS IndexName, 
OBJECT_NAME(ic.OBJECT_ID) AS TableName, 
COL_NAME(ic.OBJECT_ID,ic.column_id) AS ColumnName, 
c.is_identity
FROM sys.indexes AS i 
INNER JOIN sys.index_columns AS ic 
INNER JOIN sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id 
ON i.OBJECT_ID = ic.OBJECT_ID 
AND i.index_id = ic.index_id 
WHERE (i.is_primary_key = 1) AND ic.OBJECT_ID = OBJECT_ID('{0}')
ORDER BY OBJECT_NAME(ic.OBJECT_ID), ic.key_ordinal", GetObjectName(table));

            using (DbCommand cmd = table.GetCommand(query, con.Connection))
            {
                cmd.Transaction = con.Transaction;
                using (DbDataReader r = cmd.ExecuteReader())
                {
                    while (r.Read())
                    {
                        toReturn.Add((string)r["ColumnName"]);
                    }

                    r.Close();
                }
            }


            return(toReturn.ToArray());
        }
Exemple #12
0
        public override void MakeDistinct(DatabaseOperationArgs args, DiscoveredTable discoveredTable)
        {
            var syntax = discoveredTable.GetQuerySyntaxHelper();

            string sql =
                @"DELETE f
            FROM (
            SELECT	ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {0}) AS RowNum
            FROM {1}
            
            ) as f
            where RowNum > 1";

            string columnList = string.Join(",",
                                            discoveredTable.DiscoverColumns().Select(c => syntax.EnsureWrapped(c.GetRuntimeName())));

            string sqlToExecute = string.Format(sql, columnList, discoveredTable.GetFullyQualifiedName());

            var server = discoveredTable.Database.Server;

            using (var con = args.GetManagedConnection(server))
            {
                using (var cmd = server.GetCommand(sqlToExecute, con))
                    args.ExecuteNonQuery(cmd);
            }
        }
Exemple #13
0
        protected ICatalogue Import(DiscoveredTable tbl, out ITableInfo tableInfoCreated, out ColumnInfo[] columnInfosCreated)
        {
            CatalogueItem[]         catalogueItems;
            ExtractionInformation[] extractionInformations;

            return(Import(tbl, out tableInfoCreated, out columnInfosCreated, out catalogueItems, out extractionInformations));
        }
Exemple #14
0
 /// <summary>
 /// Returns true if the TableInfo is a reference to the discovered live table (same database, same table name, same server)
 /// <para>By default servername is not checked since you can have server aliases e.g. localhost\sqlexpress could be the same as 127.0.0.1\sqlexpress</para>
 /// </summary>
 /// <param name="discoveredTable">Pass true to also check the servername is EXACTLY the same (dangerous due to the fact that servers can be accessed by hostname or IP etc)</param>
 /// <param name="alsoCheckServer"></param>
 /// <returns></returns>
 public bool Is(DiscoveredTable discoveredTable, bool alsoCheckServer = false)
 {
     return(GetRuntimeName().Equals(discoveredTable.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase) &&
            GetDatabaseRuntimeName().Equals(discoveredTable.Database.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase) &&
            DatabaseType == discoveredTable.Database.Server.DatabaseType &&
            (!alsoCheckServer || discoveredTable.Database.Server.Name.Equals(Server, StringComparison.CurrentCultureIgnoreCase)));
 }
Exemple #15
0
        protected virtual void AddValidFrom(DiscoveredTable table, IQuerySyntaxHelper syntaxHelper)
        {
            var dateTimeDatatype = syntaxHelper.TypeTranslater.GetSQLDBTypeForCSharpType(new DatabaseTypeRequest(typeof(DateTime)));
            var nowFunction      = syntaxHelper.GetScalarFunctionSql(MandatoryScalarFunctions.GetTodaysDate);

            _table.AddColumn(SpecialFieldNames.ValidFrom, string.Format(" {0} DEFAULT {1}", dateTimeDatatype, nowFunction), true, UserSettings.ArchiveTriggerTimeout);
        }
        private string[] ListPrimaryKeys(IManagedConnection con, DiscoveredTable table)
        {
            string query = $@"SELECT               
            pg_attribute.attname, 
            format_type(pg_attribute.atttypid, pg_attribute.atttypmod) 
            FROM pg_index, pg_class, pg_attribute 
            WHERE 
            pg_class.oid = '{table.GetFullyQualifiedName()}'::regclass AND 
                indrelid = pg_class.oid AND  
            pg_attribute.attrelid = pg_class.oid AND 
            pg_attribute.attnum = any(pg_index.indkey)
            AND indisprimary";

            List <string> toReturn = new List <string>();

            using (DbCommand cmd = table.GetCommand(query, con.Connection))
            {
                cmd.Transaction = con.Transaction;

                using (DbDataReader r = cmd.ExecuteReader())
                {
                    while (r.Read())
                    {
                        toReturn.Add((string)r["attname"]);
                    }

                    r.Close();
                }
            }

            return(toReturn.ToArray());
        }
Exemple #17
0
        private void AdjustImporter()
        {
            var cataRepo = Activator.RepositoryLocator.CatalogueRepository;

            try
            {
                DiscoveredTable tbl = serverDatabaseTableSelector1.GetDiscoveredTable();

                if (tbl == null)
                {
                    btnImport.Enabled = false;
                    return;
                }

                //if it isn't a table valued function
                if (tbl is DiscoveredTableValuedFunction)
                {
                    Importer = new TableValuedFunctionImporter(cataRepo, (DiscoveredTableValuedFunction)tbl, (DataAccessContext)ddContext.SelectedValue);
                }
                else
                {
                    Importer = new TableInfoImporter(cataRepo, tbl, (DataAccessContext)ddContext.SelectedValue);
                }

                btnImport.Enabled = true;
            }
            catch (Exception exception)
            {
                ExceptionViewer.Show(exception);
            }
        }
Exemple #18
0
        protected override void SetUp()
        {
            base.SetUp();

            var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);

            parentDir = workingDir.CreateSubdirectory("FlatFileAttacherTests");

            DirectoryInfo toCleanup = parentDir.GetDirectories().SingleOrDefault(d => d.Name.Equals("Test_CSV_Attachment"));

            if (toCleanup != null)
            {
                toCleanup.Delete(true);
            }

            LoadDirectory = LoadDirectory.CreateDirectoryStructure(parentDir, "Test_CSV_Attachment");

            // create a separate builder for setting an initial catalog on (need to figure out how best to stop child classes changing ServerICan... as this then causes TearDown to fail)
            _database = GetCleanedServer(DatabaseType.MicrosoftSQLServer);

            using (var con = _database.Server.GetConnection())
            {
                con.Open();

                var cmdCreateTable = _database.Server.GetCommand("CREATE Table " + _database.GetRuntimeName() + "..Bob([name] [varchar](500),[name2] [varchar](500))", con);
                cmdCreateTable.ExecuteNonQuery();
            }

            _table = _database.ExpectTable("Bob");
        }
        protected AlterTableCommandExecution(IBasicActivateItems activator, ITableInfo tableInfo) : base(activator)
        {
            TableInfo = tableInfo;
            try
            {
                Table = TableInfo.Discover(DataAccessContext.InternalDataProcessing);
            }
            catch (Exception)
            {
                SetImpossible("Could not resolve Server/Table connection details");
                return;
            }


            if (!Table.Exists())
            {
                SetImpossible("Table does not exist");
                return;
            }

            if (Table.TableType != TableType.Table)
            {
                SetImpossible("Table is a " + Table.TableType);
                return;
            }
        }
Exemple #20
0
        public override int Run()
        {
            DiscoveredTable tbl    = GetServer(_opts.DatabaseConnectionString, _opts.DatabaseType, _opts.TableName);
            var             server = tbl.Database.Server;

            _factory = new DatabaseFailureFactory(tbl);

            _columns       = tbl.DiscoverColumns();
            _columnsNames  = _columns.Select(c => c.GetRuntimeName()).ToArray();
            _stringColumns = _columns.Select(c => c.GetGuesser().Guess.CSharpType == typeof(string)).ToArray();

            using (var con = server.GetConnection())
            {
                con.Open();

                var cmd = server.GetCommand(
                    string.Format("SELECT {0} FROM {1}"
                                  , string.Join("," + Environment.NewLine, _columns.Select(c => c.GetFullyQualifiedName()).ToArray())
                                  , tbl.GetFullyQualifiedName()), con);

                _logger.Info("About to send command:" + Environment.NewLine + cmd.CommandText);

                var reader = cmd.ExecuteReader();

                foreach (Failure failure in reader.Cast <DbDataRecord>().SelectMany(GetFailuresIfAny))
                {
                    AddToReports(failure);
                }

                CloseReports();
            }
            return(0);
        }
        public void TestBulkInsert_ExplicitDateTimeFormats(DatabaseType type)
        {
            DiscoveredDatabase db  = GetTestDatabase(type);
            DiscoveredTable    tbl = db.CreateTable("MyDateTestTable",
                                                    new[]
            {
                new DatabaseColumnRequest("MyDate", new DatabaseTypeRequest(typeof(DateTime)))
                {
                    AllowNulls = false
                },
            });

            //There are no rows in the table yet
            Assert.AreEqual(0, tbl.GetRowCount());

            using (var dt = new DataTable())
            {
                dt.Columns.Add("MyDate");
                dt.Rows.Add("20011230");

                using (IBulkCopy bulk = tbl.BeginBulkInsert())
                {
                    bulk.Timeout = 30;
                    bulk.DateTimeDecider.Settings.ExplicitDateFormats = new [] { "yyyyMMdd" };
                    bulk.Upload(dt);
                }
            }

            var dtDown = tbl.GetDataTable();

            Assert.AreEqual(new DateTime(2001, 12, 30), dtDown.Rows[0]["MyDate"]);
        }
Exemple #22
0
        /// <summary>
        /// Returns SQL for updating the <paramref name="table"/> to redact the capture groups in <see cref="IsIdentifiableRule.IfPattern"/>.  If no capture groups are represented in the <paramref name="usingRule"/> then this class falls back on <see cref="ProblemValuesUpdateStrategy"/>
        /// </summary>
        /// <param name="table"></param>
        /// <param name="primaryKeys"></param>
        /// <param name="failure"></param>
        /// <param name="usingRule"></param>
        /// <returns></returns>
        public override IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule)
        {
            if (usingRule == null || string.IsNullOrWhiteSpace(usingRule.IfPattern))
            {
                return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
            }

            try
            {
                Regex r     = new Regex(usingRule.IfPattern);
                var   match = r.Match(failure.ProblemValue);

                //Group 1 (index 0) is always the full match, we want selective updates
                if (match.Success && match.Groups.Count > 1)
                {
                    var syntax = table.GetQuerySyntaxHelper();

                    //update the capture groups of the Regex
                    return(match.Groups.Cast <Group>().Skip(1).Select(m => GetUpdateWordSql(table, primaryKeys, syntax, failure, m.Value)));
                }

                //The Regex did not have capture groups or did not match the failure
                return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
            }
            catch (Exception)
            {
                //The Regex pattern was bad or something else bad went wrong
                return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
            }
        }
Exemple #23
0
        private void DropColumnFromTable(DiscoveredTable table, string columnName, IDataLoadEventListener listener)
        {
            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, string.Format("Dropping column '{0}' from table '{1}'", columnName, table.GetFullyQualifiedName())));
            var col = table.DiscoverColumn(columnName);

            table.DropColumn(col);
        }
        protected override string GetRenameTableSql(DiscoveredTable discoveredTable, string newName)
        {
            var syntax = new PostgreSqlSyntaxHelper();

            return(@"ALTER TABLE " + discoveredTable.GetFullyQualifiedName() + @"
                RENAME TO " + syntax.EnsureWrapped(newName));
        }
Exemple #25
0
        public void CloneTable(DiscoveredDatabase srcDatabaseInfo, DiscoveredDatabase destDatabaseInfo, DiscoveredTable sourceTable, string destTableName, bool dropHICColumns, bool dropIdentityColumns, bool allowNulls, PreLoadDiscardedColumn[] dillutionColumns)
        {
            if (!sourceTable.Exists())
            {
                throw new Exception("Table " + sourceTable + " does not exist on " + srcDatabaseInfo);
            }


            //new table will start with the same name as the as the old scripted one
            DiscoveredTable newTable = destDatabaseInfo.ExpectTable(destTableName);

            var sql = sourceTable.ScriptTableCreation(allowNulls, allowNulls, false /*False because we want to drop these columns entirely not just flip to int*/, newTable);

            using (var con = destDatabaseInfo.Server.GetConnection())
            {
                con.Open();
                var cmd = destDatabaseInfo.Server.GetCommand(sql, con);
                cmd.ExecuteNonQuery();
            }

            if (!newTable.Exists())
            {
                throw new Exception("Table '" + newTable + "' not found in " + destDatabaseInfo + " despite running table creation SQL!");
            }

            foreach (DiscoveredColumn column in newTable.DiscoverColumns())
            {
                bool drop    = false;
                var  colName = column.GetRuntimeName();

                if (column.IsAutoIncrement)
                {
                    drop = true;
                }

                if (SpecialFieldNames.IsHicPrefixed(colName) && dropHICColumns)
                {
                    drop = true;
                }

                //drop the data load run ID field and validFrom fields, we don't need them in STAGING or RAW, it will be hard coded in the MERGE migration with a fixed value anyway.
                if (colName.Equals(SpecialFieldNames.DataLoadRunID) || colName.Equals(SpecialFieldNames.ValidFrom))
                {
                    drop = true;
                }

                var dillution = dillutionColumns.SingleOrDefault(c => c.GetRuntimeName().Equals(colName));

                if (dillution != null)
                {
                    column.DataType.AlterTypeTo(dillution.Data_type);
                }

                if (drop)
                {
                    newTable.DropColumn(column);
                }
            }
        }
Exemple #26
0
        private void LoadFile(DiscoveredTable tableToLoad, FileInfo fileToLoad, DiscoveredDatabase dbInfo, Stopwatch timer, IDataLoadJob job)
        {
            using (var con = dbInfo.Server.GetConnection())
            {
                DataTable dt = tableToLoad.GetDataTable(0);

                using (var insert = tableToLoad.BeginBulkInsert(Culture))
                {
                    // setup bulk insert it into destination
                    insert.Timeout = 500000;

                    //bulk insert ito destination
                    job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to open file " + fileToLoad.FullName));
                    OpenFile(fileToLoad, job);

                    //confirm the validity of the headers
                    ConfirmFlatFileHeadersAgainstDataTable(dt, job);

                    con.Open();

                    //now we will read data out of the file in batches
                    int batchNumber         = 1;
                    int maxBatchSize        = 10000;
                    int recordsCreatedSoFar = 0;

                    try
                    {
                        //while there is data to be loaded into table
                        while (IterativelyBatchLoadDataIntoDataTable(dt, maxBatchSize) != 0)
                        {
                            DropEmptyColumns(dt);
                            ConfirmFitToDestination(dt, tableToLoad, job);
                            try
                            {
                                recordsCreatedSoFar += insert.Upload(dt);

                                dt.Rows.Clear(); //very important otherwise we add more to the end of the table but still insert last batches records resulting in exponentially multiplying upload sizes of duplicate records!

                                job.OnProgress(this,
                                               new ProgressEventArgs(dbInfo.GetRuntimeName(),
                                                                     new ProgressMeasurement(recordsCreatedSoFar, ProgressType.Records), timer.Elapsed));
                            }
                            catch (Exception e)
                            {
                                throw new Exception("Error processing batch number " + batchNumber + " (of batch size " + maxBatchSize + ")", e);
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        throw new FlatFileLoadException("Error processing file " + fileToLoad, e);
                    }
                    finally
                    {
                        CloseFile();
                    }
                }
            }
        }
Exemple #27
0
        /// <inheritdoc cref="TriggerImplementer(DiscoveredTable,bool)"/>
        public PostgreSqlTriggerImplementer(DiscoveredTable table, bool createDataLoadRunIDAlso) : base(table, createDataLoadRunIDAlso)
        {
            _schema             = string.IsNullOrWhiteSpace(_table.Schema) ? table.GetQuerySyntaxHelper().GetDefaultSchemaIfAny():_table.Schema;
            _triggerRuntimeName = _table.GetRuntimeName() + "_OnUpdate";

            _procedureRuntimeName        = _table.GetRuntimeName() + "_OnUpdateProc";
            _procedureNameFullyQualified = _schema + ".\"" + _procedureRuntimeName + "\"";
        }
Exemple #28
0
 public MicrosoftSQLBulkCopy(DiscoveredTable targetTable, IManagedConnection connection, CultureInfo culture) : base(targetTable, connection, culture)
 {
     _bulkcopy = new SqlBulkCopy((SqlConnection)connection.Connection, SqlBulkCopyOptions.KeepIdentity, (SqlTransaction)connection.Transaction)
     {
         BulkCopyTimeout      = 50000,
         DestinationTableName = targetTable.GetFullyQualifiedName()
     };
 }
Exemple #29
0
        private void CheckTriggerIntact(DiscoveredTable table, ICheckNotifier notifier, out bool runSynchronizationAgain)
        {
            TriggerChecks checker = new TriggerChecks(table);

            checker.Check(notifier);

            runSynchronizationAgain = checker.TriggerCreated;
        }
Exemple #30
0
        private TableInfo ImportTableInfo(DiscoveredTable tbl)
        {
            var importer = new TableInfoImporter(_repos.CatalogueRepository, tbl);

            importer.DoImport(out TableInfo ti, out _);

            return(ti);
        }