public void IsLoadProcessKeySetForLogTask()
{
//Arrange
ControlFlow.DefaultDbConnection = SqlConnection;
CreateLoadProcessTableTask.Create("test_lp_logtask");
CreateLogTableTask.Create("test_log_logtask");
ControlFlow.AddLoggingDatabaseToConfig(SqlConnection, NLog.LogLevel.Info, "test_log_logtask");
//Act
StartLoadProcessTask.Start("Test process 15");
long?processId1 = ControlFlow.CurrentLoadProcess.Id;
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
//Assert
Assert.Equal(3, new RowCountTask("test_log_logtask",
$"message like 'Test%' AND load_process_id = {processId1}")
{
DisableLogging = true
}.Count().Rows);
//Cleanup
DropTableTask.Drop(ControlFlow.LogTable);
DropTableTask.Drop(ControlFlow.LoadProcessTable);
}
public void TestErrorLogging(IConnectionManager conn)
{
//Arrange
CreateLogTableTask.Create(conn, "test_log");
ControlFlow.AddLoggingDatabaseToConfig(conn, NLog.LogLevel.Trace, "test_log");
//Act
LogTask.Error(conn, "Error!");
LogTask.Warn(conn, "Warn!");
LogTask.Info(conn, "Info!");
LogTask.Debug(conn, "Debug!");
LogTask.Trace(conn, "Trace!");
LogTask.Fatal(conn, "Fatal!");
//Assert
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Error!' AND {conn.QB}level{conn.QE} = 'Error' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Warn!' AND {conn.QB}level{conn.QE} = 'Warn' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Info!' AND {conn.QB}level{conn.QE} = 'Info' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Debug!' AND {conn.QB}level{conn.QE} = 'Debug' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Trace!' AND {conn.QB}level{conn.QE} = 'Trace' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Fatal!' AND {conn.QB}level{conn.QE} = 'Fatal' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
//Cleanup
DropTableTask.Drop(conn, ControlFlow.LogTable);
}
public void TestLoggingWithOdbc()
{
ControlFlow.CurrentDbConnection = new OdbcConnectionManager(new OdbcConnectionString(OdbcConnectionStringParameter));
CreateLogTablesTask.CreateLog();
LogTask.Info("Info");
Assert.AreEqual(1, SqlTask.ExecuteScalar <int>("Check if default log works", "select count(*) from etl.Log where Message in ('Error','Warn','Info')"));
}
public void TestErrorLogging()
{
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
Assert.AreEqual(3, SqlTask.ExecuteScalar <int>("Check if default log works", "select count(*) from etl.Log where Message in ('Error','Warn','Info')"));
}
private void RunDemoProcess()
{
new Sequence("Test sequence 1", RunSubSequence)
{
TaskType = "SUBPACKAGE"
}.Execute();
SqlTask.ExecuteNonQuery($"Sql #1", "Select 1 as test");
LogTask.Info("Info message");
}
public void TestTransferAndLogging()
{
//CurrentDbConnection is always use if ConnectionManager is not specified otherwise!
ControlFlow.CurrentDbConnection = new SqlConnectionManager(ConnectionStringSource);
SqlTask.ExecuteNonQuery("Create source table", @"CREATE TABLE test.Source
(Col1 nvarchar(100) null, Col2 int null)");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1',1)");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test2',2)");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test3',3)");
ControlFlow.CurrentDbConnection = new SqlConnectionManager(ConnectionStringLog);
CreateLogTablesTask.CreateLog();
new CreateTableTask("test.Destination", new List <ITableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
})
{
ConnectionManager = new SqlConnectionManager(ConnectionStringDest)
}.Execute();
var sourceConnection = new SqlConnectionManager(new ConnectionString(ConnectionStringSource));
var destConnection = new SqlConnectionManager(new ConnectionString(ConnectionStringDest));
DBSource source = new DBSource(sourceConnection, "test.Source");
RowTransformation trans = new RowTransformation(row =>
{
LogTask.Info($"Test message: {row[0]}, {row[1]}"); //Log DB is used as this is the ControlFlow.CurrentDBConnection!
return(row);
});
DBDestination destination = new DBDestination(destConnection, "test.Destination");
source.LinkTo(trans);
trans.LinkTo(destination);
source.Execute();
destination.Wait();
Assert.AreEqual(1, new RowCountTask("test.Destination", "Col1 = 'Test1' AND Col2=1")
{
ConnectionManager = new SqlConnectionManager(ConnectionStringDest)
}.Count().Rows);
Assert.AreEqual(1, new RowCountTask("test.Destination", "Col1 = 'Test2' AND Col2=2")
{
ConnectionManager = new SqlConnectionManager(ConnectionStringDest)
}.Count().Rows);
Assert.AreEqual(1, new RowCountTask("test.Destination", "Col1 = 'Test3' AND Col2=3")
{
ConnectionManager = new SqlConnectionManager(ConnectionStringDest)
}.Count().Rows);
}
public void TestErrorLogging()
{
//Arrange
//Act
LogTask.Error(Connection, "Error");
LogTask.Warn(Connection, "Warn");
LogTask.Info(Connection, "Info");
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "etl.Log",
"Message in ('Error','Warn','Info')"));
}
public void TestLogCleanup()
{
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
CleanUpLogTask.CleanUp(0);
Assert.AreEqual(0, new SqlTask("Check if log table is empty", $"select count(*) from etl.Log ")
{
DisableLogging = true
}.ExecuteScalar <int>());
}
public void BigData_CSV_ACCESS_ViaOdbc()
{
int numberOfRows = 2000;
ControlFlow.CurrentDbConnection = new AccessOdbcConnectionManager(new OdbcConnectionString(AccessConnectionStringParameter))
{
AlwaysUseSameConnection = false
};
Stopwatch watch = new Stopwatch();
TableDefinition stagingTable = new TableDefinition("staging", new List <TableColumn>()
{
new TableColumn("Col1", "CHAR", allowNulls: true),
new TableColumn("Col2", "CHAR", allowNulls: true),
new TableColumn("Col3", "CHAR", allowNulls: false),
new TableColumn("Col4", "CHAR", allowNulls: false),
});
try {
SqlTask.ExecuteNonQuery("Try to drop table", $@"DROP TABLE {stagingTable.Name};");
} catch { }
new CreateTableTask(stagingTable)
{
ThrowErrorIfTableExists = true
}.Execute();
string fileName = "src/ConnectionManager/AccessBigData_CSV2DB.csv";
BigDataHelper bigData = new BigDataHelper()
{
FileName = fileName,
NumberOfRows = numberOfRows,
TableDefinition = stagingTable
};
watch.Start();
LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows");
bigData.CreateBigDataCSV();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file");
watch.Reset();
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(30)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
dest.Wait();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database");
Assert.AreEqual(numberOfRows, RowCountTask.Count(stagingTable.Name));
}
public void TestLoadProcessKeyForLogTask()
{
StartLoadProcessTask.Start("Test process 8");
int?processKey1 = ControlFlow.CurrentLoadProcess.LoadProcessKey;
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
Assert.AreEqual(3, new SqlTask("Check if load process key is set", $"select count(*) from etl.Log where Message like 'Test%' and LoadProcessKey = {processKey1}")
{
DisableLogging = true
}.ExecuteScalar <int>());
}
private static void StartDefaultCSVLoad(Stopwatch watch, TableDefinition stagingTable, string fileName)
{
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(1000)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to read everything into memory (while constantly writing)");
LogTask.Info($"Already {RowCountTask.Count("test.Staging", RowCountOptions.QuickQueryMode)} inserted into table");
dest.Wait();
}
public void LogCleanup()
{
//Arrange
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
//Act
CleanUpLogTask.CleanUp(Connection, 0);
//Assert
Assert.Equal(0, new RowCountTask("etl.Log ")
{
DisableLogging = true,
ConnectionManager = Connection
}.Count().Rows);
}
public void IsLoadProcessKeySetForLogTask()
{
//Arrange
StartLoadProcessTask.Start("Test process 15");
int?processKey1 = ControlFlow.CurrentLoadProcess.LoadProcessKey;
//Act
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
//Assert
Assert.Equal(3, new RowCountTask("etl.Log",
$"Message like 'Test%' AND LoadProcessKey = {processKey1}")
{
DisableLogging = true
}.Count().Rows);
}
public void BigData_CSV_DB(int numberOfRows)
{
Stopwatch watch = new Stopwatch();
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nchar(1000)", allowNulls: false),
new TableColumn("Col2", "nchar(1000)", allowNulls: false),
new TableColumn("Col3", "nchar(1000)", allowNulls: false),
new TableColumn("Col4", "nchar(1000)", allowNulls: false),
});
stagingTable.CreateTable();
string fileName = "src/DataFlow/BigData_CSV2DB.csv";
BigDataHelper bigData = new BigDataHelper()
{
FileName = fileName,
NumberOfRows = numberOfRows,
TableDefinition = stagingTable
};
watch.Start();
LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows");
bigData.CreateBigDataCSV();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file");
watch.Reset();
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(1000)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to read everything into memory (while constantly writing)");
LogTask.Info($"Already {RowCountTask.Count("test.Staging", RowCountOptions.QuickQueryMode)} inserted into table");
dest.Wait(); //TODO Wait should be part of source
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database");
Assert.AreEqual(numberOfRows, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging"));
}
public void BigData_CSV_DB(int numberOfRows, bool useGenericCSVSource = false)
{
Stopwatch watch = new Stopwatch();
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nchar(1000)", allowNulls: false),
new TableColumn("Col2", "nchar(1000)", allowNulls: false),
new TableColumn("Col3", "nchar(1000)", allowNulls: false),
new TableColumn("Col4", "nchar(1000)", allowNulls: false),
});
stagingTable.CreateTable();
string fileName = "src/DataFlow/BigData_CSV2DB.csv";
BigDataHelper bigData = new BigDataHelper()
{
FileName = fileName,
NumberOfRows = numberOfRows,
TableDefinition = stagingTable
};
watch.Start();
LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows");
bigData.CreateBigDataCSV();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file");
watch.Reset();
if (useGenericCSVSource)
{
StartGenericCSVLoad(watch, stagingTable, fileName);
}
else
{
StartDefaultCSVLoad(watch, stagingTable, fileName);
}
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database");
Assert.AreEqual(numberOfRows, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging"));
}
public void Start()
{
ControlFlow.STAGE = "0";
var logger = new LogTask()
{
Message = "D_Customer Task",
ActionType = "Start"
};
logger.Info();
//Get Last CheckTime
ControlFlow.STAGE = (int.Parse(ControlFlow.STAGE) + 1) + "";
string lastCheckTime = CRMDBManager.GetLastCheckTime("D_CUSTOMER");
//Truncate Table ETL_TEMP_Profiles
//PMSDBManager.TruncateTable("ETL_TEMP_Profiles");
//TRUNCATE TABLE ETL_TEMP_PROFILES_D_CUSTOMER;
//TRUNCATE TABLE ETL_TEMP_PROFILES_D_CUSTOMER_INSERT;
//TRUNCATE TABLE ETL_TEMP_PROFILES_D_CUSTOMER_UPDATE;
//TRUNCATE TABLE ETL_TEMP_D_CUSTOMER_For_Email;
//CRMDBManager.TruncateTable("ETL_TEMP_PROFILES_D_CUSTOMER", "ETL_TEMP_PROFILES_D_CUSTOMER_INSERT", "ETL_TEMP_PROFILES_D_CUSTOMER_UPDATE", "ETL_TEMP_D_CUSTOMER_For_Email");
var companyId = CompanySettings.CompanyID;
if ("7375".Equals(companyId))
{
// Create Source Code For Open Table
string openTableSourceCode = CRMDBManager.CreateSourceCode(companyId, "CLIENT", "Open Table", 1, 99, 99);//SQLHelper.GetDbValues(strConnectionString, "sqlCreateSourceCodeForOpenTable", sqlCreateSourceCodeForOpenTable, null);
// Create Source Code For Counter Point
string counterPointSourceCode = CRMDBManager.CreateSourceCode(companyId, "CLIENT", "Counter Point", 1, 99, 99);//SQLHelper.GetDbValues(strConnectionString, "sqlCreateSourceCodeForCounterPoint", sqlCreateSourceCodeForCounterPoint, null);
}
//create source code
string pmsSourceCode = CRMDBManager.CreateSourceCode(companyId, "PMS", "PMS", 0, 1, 1);//SQLHelper.GetDbValues(strConnectionString, "sqlCreateSourceCode", sqlCreateSourceCode, null);
// Move ProfileDocument
CustomerDataFlowTaskManager.DFT_MoveProfileDocument();
// Move ProfilePolicies
CustomerDataFlowTaskManager.DFT_MoveProfilePolicies();
//Move ContactMethodPolicies
CustomerDataFlowTaskManager.DFT_MoveContactMethodPolicies();
//Move ContactMethod
CustomerDataFlowTaskManager.DFT_MoveContactMethod();
//Update SourceName under ContactMethod
CustomerDataFlowTaskManager.DFT_UpdateSourceNameUnderContactMethod();
//Move Action comments
CustomerDataFlowTaskManager.DFT_MoveActionComments();
//Move Address
CustomerDataFlowTaskManager.DFT_MoveAddress();
//Update AddressTypeCode under table PMS_Address
CRMDBManager.UpdateAddressTypeCodeUnderPMS_Address();
//Move special Requests - Omni
if ("12123".Equals(companyId) || "13358".Equals(companyId) || "13298".Equals(companyId))
{
CustomerDataFlowTaskManager.DFT_MoveSpecialRequests_Omni();
}
else
{
//Truncate CENRES_Specialrequests
CRMDBManager.TruncateTable("CENRES_SpecialRequests", "ETL_TEMP_Remove_SpecialRequests");
//Move Updated Profiles into Temp table
CustomerDataFlowTaskManager.DFT_MoveUpdatedProfilesIntoTempTable();
//Delete Global Special Requests for Updated Profiles
CRMDBManager.DeleteGlobalSpecialRequestsForUpdatedProfiles();
//Move the existing Special Requests
//Move Existing Special Requests
CRMDBManager.MoveExistingSpecialRequests();
//Move Special Requests
CustomerDataFlowTaskManager.DFT_MoveSpecialRequests_Omni();
//Remove ResortField for Global Request
CRMDBManager.RemoveResortFieldForGlobalRequest();
//Truncate CENRES_Specialrequests
CRMDBManager.TruncateTable("CENRES_SpecialRequests");
//Move Special Requests into temp table
//Delete Orphan records that don't match CenRes from PMS_SpecialRequests table
CRMDBManager.DeleteOrphanRecords();
}
if ("7375".Equals(companyId))
{
// Update PMS Profile Mapping for Biltmore
CRMDBManager.UpdatePMSProfileMappingForBiltmore();
}
else
{
//Update PMS Profile Mapping
CRMDBManager.UpdatePMSProfileMapping();
}
// Move the New or Updates in tehe profiles in the Temp Table
// Separate the Insert and Update into 2 temp tables
//Insert profiles in D_Customer without membership
CustomerDataFlowTaskManager.DFT_InsertOrUpdateD_CustomerWithoutMembership();
if ("1338".Equals(companyId) || "1475".Equals(companyId) || "6245".Equals(companyId) || "7902".Equals(companyId) || "8009".Equals(companyId) || "7692".Equals(companyId) || "11733".Equals(companyId) || "11761".Equals(companyId))
{
//Update profiles into D_Customer with membership
CustomerDataFlowTaskManager.DFT_InsertOrUpdateD_CustomerWithMembershipColumnUpdate();
}
else
{
//Update profiles into D_Customer without membership Column Update
CustomerDataFlowTaskManager.DFT_InsertOrUpdateD_CustomerWithoutMembershipColumnUpdate();
}
//Update property code
CustomerDataFlowTaskManager.DFT_UpdatePropertyCode();
//issue Move Profiles to PMS_Profiles and PMS_PROFILE_MAPPING
CustomerDataFlowTaskManager.DFT_MoveProfilesToPMS_ProfilesAndPMS_PROFILE_MAPPING();
//Move Profiles_Ext to PMS_Profiles_Ext
CustomerDataFlowTaskManager.DFT_MoveProfiles_ExtToPMS_Profiles_Ext();
//Move Birthday into D_Customer_Profile
CustomerDataFlowTaskManager.DFT_MoveBirthday();
//Move Anniversary into D_Customer_Profile
CustomerDataFlowTaskManager.DFT_MoveAnniversaryIntoD_Customer_Profile();
//Move SpouseBirthDate into D_Customer_Profile
CustomerDataFlowTaskManager.DFT_MoveSpouseBirthDateIntoD_Customer_Profile();
if ("11312".Equals(companyId))
{
//Update Preferred Language in PMS_Profiles table - Minor
CustomerDataFlowTaskManager.DFT_UpdatePreferredLanguageUnderPMS_Profiles();
//Update preferred language in D_Customer table - Minor
CRMDBManager.UpdatePreferredlanguageUnderD_Customer();
}
else
{
//Truncate temp tables
CRMDBManager.TruncateTable("ETL_TEMP_D_CUSTOMER_PHONE");
}
if ("12123".Equals(companyId) || "13298".Equals(companyId) || "13358".Equals(companyId))
{
//Update all address fields to blank - Omni
CRMDBManager.UpdateAllAddressFieldsToBlank();
}
//Update D_Customer table for Address
CustomerDataFlowTaskManager.DFT_UpdateD_CustomerForAddress();
if ("7375".Equals(companyId))
{
// Update Email in D_Customer for Biltmore
CustomerDataFlowTaskManager.DFT_UpdateEmailUnderD_Customer();
// Update Phone Numbers in D_Customer for Biltmore
CustomerDataFlowTaskManager.DFT_UpdatePhoneNumbersUnderD_Customer();
// Update Phone Ext in D_Customer for Biltmore
CustomerDataFlowTaskManager.DFT_UpdatePhoneExtUnderD_Customer();
}
else
{
//Remove inactive email records
CustomerDataFlowTaskManager.DFT_RemoveInactiveEmailRecords();
if ("10738".Equals(companyId))
{
//Update AddressType as W if it is U for Rydges under D_Customer table
CRMDBManager.UpdateAddressTypeUnderD_Customer();
}
//Update Email And Phone In D_Customer
CustomerDataFlowTaskManager.DFT_UpdateEmailAndPhoneUnderD_Customer();
if ("12123".Equals(companyId) || "13298".Equals(companyId) || "13358".Equals(companyId))
{
//Update all phone fields to blank - Omni
CRMDBManager.UpdateAllPhoneFieldsToBlank();
}
//Update Phones in D_Customer
CRMDBManager.UpdatePhonesUnderD_Customer();
}
// D_Customer Email maintenance
CRMDBManager.D_CustomerEmailMaintenance();
if ("6543".Equals(companyId))
{
//Update D_Customer_Email Rosewood
CustomerDataFlowTaskManager.DFT_UpdateD_Customer_Email();
}
//Fill Email table
//Move data from ETL_D_Customer_Email_Staging table to D_Customer_Email table
CustomerDataFlowTaskManager.DFT_UpSertDataIntoD_Customer_Email();
//Truncate table ETL_D_CUSTOMER_EMAIL_STAGING
CRMDBManager.TruncateTable("ETL_D_CUSTOMER_EMAIL_STAGING");
if ("10960".Equals(companyId))
{
//Update OptIn OptOut based on ProfilePolicies Data
CRMDBManager.UpdateOptInOptOut();
}
//Move EmailList type of data
CustomerDataFlowTaskManager.DFT_MoveEmailListTypeOfData();
//Move UDF31 Type Of Data
CustomerDataFlowTaskManager.DFT_MoveUDF31TypeOfData();
//Update VIPLevel based on UDFC31 type of data
CRMDBManager.UpdateVIPLevel();
if ("11757".Equals(companyId) || "12988".Equals(companyId) || "13006".Equals(companyId))
{
//Move KanaLastName
CustomerDataFlowTaskManager.DFT_MoveKanaLastName();
//Move KanaFirstName
CustomerDataFlowTaskManager.DFT_MoveKanaFirstName();
//Move NKanaLastName
CustomerDataFlowTaskManager.DFT_MoveNKanaLastName();
//Move NKanaFirstName
CustomerDataFlowTaskManager.DFT_MoveNKanaFirstName();
//Move NKanaName
CustomerDataFlowTaskManager.DFT_MoveNKanaName();
//Move UDFC37 Type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC37TypeOfData();
}
else if ("12123".Equals(companyId) || "13298".Equals(companyId) || "13358".Equals(companyId))
{
//Move Always_Email_Folio data
CustomerDataFlowTaskManager.DFT_MoveAlways_Email_DolioData();
//Move GHA_Email Data
CustomerDataFlowTaskManager.DFT_MoveGHA_EmailData();
//Move TOTEL_OFFER_EMAIL Data
CustomerDataFlowTaskManager.DFT_MoveHotel_Offer_EmailData();
//Move Marketing_Email Data
CustomerDataFlowTaskManager.DFT_MoveMarketing_EmailData();
//Move Marketing_Print Data
CustomerDataFlowTaskManager.DFT_MoveMarketing_PrintData();
//Move Mokara_Email Data
CustomerDataFlowTaskManager.DFT_MoveMokara_EmailData();
//Move New_Card_Request Data
CustomerDataFlowTaskManager.DFT_MoveNew_Card_RequestData();
//Move NewsLetter_Email Data
CustomerDataFlowTaskManager.DFT_MoveNewsLetter_EmailData();
//Move SG_ACCOUNT_SUMMARY Data
CustomerDataFlowTaskManager.DFT_MoveSG_Account_SummaryData();
//Move Transaction_Email Data
CustomerDataFlowTaskManager.DFT_MoveTransaction_EmailData();
//Move SG_Email Data
CustomerDataFlowTaskManager.DFT_MoveSG_EmailData();
//Move UDFData
CustomerDataFlowTaskManager.DFT_MoveUDFData();
}
else
{
//Move UDFC37 Type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC37TypeOfData();
}
if ("7375".Equals(companyId))
{
//Move Sales Rep UDF Data
CustomerDataFlowTaskManager.DFT_MoveSalesRepUDFData();
//Move Est Rate UDF Data
CustomerDataFlowTaskManager.DFT_MoveEstrateUDFData();
//Move Depart UDF Data
CustomerDataFlowTaskManager.DFT_MoveDepartUDFData();
//Move Arrival UDF Data
CustomerDataFlowTaskManager.DFT_MoveArrivalUDFData();
//Move Block Ref UDF Data
CustomerDataFlowTaskManager.DFT_MoveBlockRefUDFData();
//Move Market Segment UDF Data
CustomerDataFlowTaskManager.DFT_MoveMarketSegmentUDFData();
//Move Market SubSegment UDF Data
CustomerDataFlowTaskManager.DFT_MoveMarketSubSegmentUDFData();
//Move Source UDF Data
CustomerDataFlowTaskManager.DFT_MoveSourceUDFData();
//Move Notes UDF Data
CustomerDataFlowTaskManager.DFT_MoveNotesUDFData();
//Move Banned UDF Data
CustomerDataFlowTaskManager.DFT_MoveBannedUDFData();
//Move Address3 UDF Data
CustomerDataFlowTaskManager.DFT_MoveAddress3UDFData();
//Move Alternate UDF Data
CustomerDataFlowTaskManager.DFT_MoveAlternateUDFData();
//Update CompanyName from ContactMethod table
CustomerDataFlowTaskManager.DFT_UpdateCompanyNameFromContactMethod();
//Move cnotes UDF Data for OpenTable
CustomerDataFlowTaskManager.DFT_MoveCNotesUDFData();
//Move count_cancel UDF Data for OpenTable
CustomerDataFlowTaskManager.DFT_MoveCount_CancelUDFData();
//Move count_noshow UDF Data for OpenTable
CustomerDataFlowTaskManager.DFT_MoveCount_NoshowUDFData();
//Move count_resos UDF Data for OpenTable
CustomerDataFlowTaskManager.DFT_MoveCount_ResosUDFData();
//Move custcodes UDF Data for OpenTable
CustomerDataFlowTaskManager.DFT_MoveCustcodesUDFData();
//Move BAPExpiration UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveBAPExpirationUDFData();
//Move ChildMembership UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveChildMembershipUDFData();
//Move Comment UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveCommentUDFData();
//Move CreatedOnline UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveCreatedOnlineUDFData();
//Move CustomerCode UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveCustomerCodeUDFData();
//Move HasOnlineProfile UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveHasOnlineProfileUDFData();
//Move LastOnlineOrderDate UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveLastOnlineOrderDateUDFData();
//Move LastOnlineOrderNumber UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveLastOnlineOrderNumberUDFData();
//Move OmniPassBarcode UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveOmniPassBarcodeUDFData();
//Move OmniPassID UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveOmniPassIDUDFData();
//Move OnlineCustomerNumber UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveOnlineCustomerNumberUDFData();
//Move ProductDelivery UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MoveProductDeliveryUDFData();
//Move PROF_COD_5 UDF Data for counterpoint
CustomerDataFlowTaskManager.DFT_MovePROF_COD_5UDFData();
}
if ("11964".Equals(companyId))
{
//Move UDFC20 type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC20TypeOfData();
//Move UDFC22 type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC22TypeOfData();
//Move UDFC23 type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC23TypeOfData();
//Move UDFC33 type of data
CustomerDataFlowTaskManager.DFT_MoveUDFC33TypeOfData();
}
//DTF Connector
//EmailVerify With Bounce Rules
CRMDBManager.EmailVerifyWithBounceRules();
// EmailVerify on D_Customer
CRMDBManager.EmailVerifyOnD_Customer();
if ("6543".Equals(companyId))
{
//EmailVerify on D_CUSTOMER_EMAIL - Rosewood
CRMDBManager.EmailVerifyOnD_CUSTOMER_EMAIL_Rosewood();
}
else
{
//Move Emails From D_Customer To D_Customer_Email _NonRosewood
CustomerDataFlowTaskManager.DFT_MoveEmailsFromD_CustomerToD_Customer_Email_NonRosewood();
//Fill up missing data under D_Customer_Email table
CRMDBManager.FillUpMissingDataUnderD_Customer_Email();
//Fix emails which are not matching under D_Customer_Email table
CRMDBManager.FixEmailsDataUnderD_Customer_Email();
//Fix email_id under D_Customer_Email table
CRMDBManager.FixEmail_IDUnderD_Customer_Email();
}
if ("12123".Equals(companyId) || "13298".Equals(companyId) || "13358".Equals(companyId))
{
//Update Inactive profiles record status - omni
CustomerDataFlowTaskManager.DFT_UpdateRecordStatusUnderPMS_Profiles();
CustomerDataFlowTaskManager.DFT_UpdateRecordStatusUnderD_Customer();
}
if ("1".Equals(CompanySettings.ETL_HASUNIFOCUSDATA))//@HasUNIFOCUSData == 1
{
//Update Unifocus score per customer
CRMDBManager.UpdateUnifocusScorePerCustomer();
}
// Insert EndTime into Log Table
//CRMDBManager.InsertEndTimeIntoLogTable();
logger.ActionType = "END";
logger.Info();
ControlFlow.STAGE = (int.Parse(ControlFlow.STAGE) - 1) + "";
}
public void runTask(string sConnString, string dConnString, string dTableName, string sql, bool isUpdate = false, bool isInsert = false, List <string> primaryKey = null, List <string> updateFields = null)
{
var logger = new LogTask()
{
Message = "Data Flow Task: " + TaskName,
ActionType = "Start"
};
logger.Info();
ControlFlow.STAGE = (int.Parse(ControlFlow.STAGE) + 1) + "";
//Type of Data Flow Task
ActionType actionType = ActionType.Upsert;
if (isInsert && isUpdate)
{
actionType = ActionType.Upsert;
}
else if (isInsert)
{
actionType = ActionType.Insert;
}
else if (isUpdate)
{
actionType = ActionType.Update;
}
else
{
//return;
}
if (actionType == ActionType.Upsert && (primaryKey == null || primaryKey.Count == 0))
{
actionType = ActionType.Insert;
}
//Execute Task Flow task
DBSource <T> dBSource = new DBSource <T>()
{
_disableLogging = false,
ConnString = sConnString,
Sql = sql
};
RowTransformation <T, T> rowT = new RowTransformation <T, T>(
input => input
);
DBDestination <T> dBDestination = new DBDestination <T>(dTableName)
{
ConnString = dConnString,
ActionType = actionType,
Keys = primaryKey,
UpdateFields = updateFields
};
dBSource.LinkTo(rowT);
rowT.LinkTo(dBDestination);
dBSource.Execute();
dBDestination.Wait();
logger.ActionType = "END";
logger.Info();
ControlFlow.STAGE = (int.Parse(ControlFlow.STAGE) - 1) + "";
}
