public void Run()
{
List <Landing_Table> import_tables = repo.GetLandingTables();
tasks.Clear();
di_execution execution_log = repo.NewExecution("Oracle data import");
foreach (Landing_Table table in import_tables)
{
OracleDataImport odi = new OracleDataImport(table, execution_log.execution_id, repo);
tasks.Add(Task.Factory.StartNew(() => DoWork(odi)));
}
Task.WaitAll(tasks.ToArray());
execution_log.status = "Finished";
repo.Update(execution_log);
}
public void ImportTable()
{
SourceTable = tableName;
DestinationTable = tableName;
string[] ignoreRowCountTables = { "SWVTRAN" };
logger.Info("Beginning landing for table {0}", Table.Table_Name);
operation = repo.NewOperation(ExecutionId, string.Format("Data import for {0}.{1}", Table.Dest_Schema, Table.Table_Name));
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "OnStart", string.Format("Import {0}.{1} beginning", Table.Dest_Schema, Table.Table_Name), "Information");
TableLog = repo.New_Log_Record(Table.id);
try
{
SourceTableCount();
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Source row count for {0}.{1} is {2}", Table.Dest_Schema, Table.Table_Name, TableLog.Source_Rows), "Information");
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Starting to land table {0}.{1}", Table.Dest_Schema, Table.Table_Name), "Information");
LandTable();
LandingTableCount();
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Finished landing {0}.{1} row count is {2}", Table.Dest_Schema, Table.Table_Name, TableLog.Landing_Rows), "Information");
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Starting add hashes for table {0}.{1}", Table.Dest_Schema, Table.Table_Name), "Information");
HashTable();
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Finished adding hashes for table {0}.{1}", Table.Dest_Schema, Table.Table_Name), "Information");
logger.Info("Rows in {0} source: {1}, landing: {2}", Table.Table_Name, TableLog.Source_Rows, TableLog.Landing_Rows);
if (TableLog.Source_Rows != TableLog.Landing_Rows)
{
if (!ignoreRowCountTables.Contains(Table.Table_Name))
{
throw new Exception("Source and landing table rows don't match, not continuing");
}
else
{
logger.Info("Exception Ignored for excluded table {0}. Rowcount exception. Landing rows: {1} Stage rows: {2}", Table.Table_Name, TableLog.Landing_Rows, TableLog.Stage_Rows);
}
}
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Starting stage for {0}.{1}", Table.Dest_Schema, Table.Table_Name), "Information");
StageTable();
StageTableCount();
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", string.Format("Finished stage for {0}.{1} row count is {2}", Table.Dest_Schema, Table.Table_Name, TableLog.Stage_Rows), "Information");
logger.Info("Rows in {0} stage: {1}", Table.Table_Name, TableLog.Stage_Rows);
if (TableLog.Stage_Rows == TableLog.Source_Rows)
{
TableLog.Success = 1;
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "Information", "Import finished and source/stage row counts match: " + TableLog.Source_Rows + ":" + TableLog.Stage_Rows + " rows", "Information");
operation.operation_status = "Success";
}
else
{
TableLog.Success = 0;
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "OnError", "Row Mismatch between source and staging table", "Error");
operation.operation_status = "Finished with row count error";
}
}
catch (Exception ex)
{
logger.Error(ex, "Exception while landing {0}: {1}", Table.Table_Name, ex.Message);
operation.operation_status = "Failed";
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "OnError", "Exception while trying to import table: " + ex.Message + " " + ex.Source + " " + ex.StackTrace, "Error");
operation.operation_status = "Failed";
}
finally
{
operation.end_time = DateTime.Now;
repo.Update_Log(TableLog);
repo.Update(operation);
logger.Info("Finished landing for table {0}", Table.Table_Name);
repo.AddMessage(operation.operation_id, operation.operation_desc + "\\ImportTable", "OnComplete", "Import finished and source/stage row counts match", "Information");
}
/*
* hashing this out for now, moving to simplistic method, revisit later
* //try
* //{
* if (GetSourceSchema())
* {
* if (GetDestSchema())
* {
* if (!SchemasMatch())
* {
* WriteToLog("info", "Schema's don't match, drop and recreating");
* DropDestTable();
* CreateDestTable();
* }
* }
* else
* {
* WriteToLog("info", "Destination Table doesn't exist, creating");
* CreateDestTable();
* }
*
* // At this point the destination table should be good to go
* GetDestSchema(); // refresh the schema
*
* DeleteAndImport();
* }
* else
* {
* WriteToLog("Error", "Source table does not exist");
* }
* //}
* //catch (Exception ex)
* //{
* // WriteToLog("Error", "Unspecified error during import: "+ex.Message);
*
* //}
*/
}