public void IsLoadProcessKeySetForLogTask()
{
//Arrange
ControlFlow.DefaultDbConnection = SqlConnection;
CreateLoadProcessTableTask.Create("test_lp_logtask");
CreateLogTableTask.Create("test_log_logtask");
ControlFlow.AddLoggingDatabaseToConfig(SqlConnection, NLog.LogLevel.Info, "test_log_logtask");
//Act
StartLoadProcessTask.Start("Test process 15");
long?processId1 = ControlFlow.CurrentLoadProcess.Id;
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
//Assert
Assert.Equal(3, new RowCountTask("test_log_logtask",
$"message like 'Test%' AND load_process_id = {processId1}")
{
DisableLogging = true
}.Count().Rows);
//Cleanup
DropTableTask.Drop(ControlFlow.LogTable);
DropTableTask.Drop(ControlFlow.LoadProcessTable);
}
public void TestErrorLogging(IConnectionManager conn)
{
//Arrange
CreateLogTableTask.Create(conn, "test_log");
ControlFlow.AddLoggingDatabaseToConfig(conn, NLog.LogLevel.Trace, "test_log");
//Act
LogTask.Error(conn, "Error!");
LogTask.Warn(conn, "Warn!");
LogTask.Info(conn, "Info!");
LogTask.Debug(conn, "Debug!");
LogTask.Trace(conn, "Trace!");
LogTask.Fatal(conn, "Fatal!");
//Assert
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Error!' AND {conn.QB}level{conn.QE} = 'Error' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Warn!' AND {conn.QB}level{conn.QE} = 'Warn' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Info!' AND {conn.QB}level{conn.QE} = 'Info' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Debug!' AND {conn.QB}level{conn.QE} = 'Debug' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Trace!' AND {conn.QB}level{conn.QE} = 'Trace' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
Assert.Equal(1, RowCountTask.Count(conn, ControlFlow.LogTable,
$"{conn.QB}message{conn.QE} = 'Fatal!' AND {conn.QB}level{conn.QE} = 'Fatal' AND {conn.QB}task_action{conn.QE} = 'LOG'"));
//Cleanup
DropTableTask.Drop(conn, ControlFlow.LogTable);
}
public void Start()
{
//Recreate database
ControlFlow.CurrentDbConnection = new SqlConnectionManager(new ConnectionString("Data Source=.;Integrated Security=SSPI;"));
DropDatabaseTask.Drop("DemoDB");
CreateDatabaseTask.Create("DemoDB");
ControlFlow.CurrentDbConnection = new SqlConnectionManager(new ConnectionString("Data Source=.;Integrated Security=SSPI;Initial Catalog=DemoDB;"));
//Logging (only works with existing configuration nlog config in App.config)
CreateLogTablesTask.CreateLog();
StartLoadProcessTask.Start("Process 1");
ControlFlow.STAGE = "Staging";
SqlTask.ExecuteNonQuery("some sql", "Select 1 as test");
TransferCompletedForLoadProcessTask.Complete();
ControlFlow.STAGE = "DataVault";
Sequence.Execute("some custom code", () => { });
LogTask.Warn("Some warning!");
EndLoadProcessTask.End("Everything successful");
string jsonLP = GetLoadProcessAsJSONTask.GetJSON();
string jsonLog = GetLogAsJSONTask.GetJSON(1);
}
public void TestErrorLogging()
{
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
Assert.AreEqual(3, SqlTask.ExecuteScalar <int>("Check if default log works", "select count(*) from etl.Log where Message in ('Error','Warn','Info')"));
}
public void TestLogCleanup()
{
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
CleanUpLogTask.CleanUp(0);
Assert.AreEqual(0, new SqlTask("Check if log table is empty", $"select count(*) from etl.Log ")
{
DisableLogging = true
}.ExecuteScalar <int>());
}
public void TestErrorLogging()
{
//Arrange
//Act
LogTask.Error(Connection, "Error");
LogTask.Warn(Connection, "Warn");
LogTask.Info(Connection, "Info");
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "etl.Log",
"Message in ('Error','Warn','Info')"));
}
public static TimeSpan LogExecutionTime(string name, Action action)
{
Stopwatch watch = new Stopwatch();
LogTask.Warn($"Starting: {name}");
watch.Start();
action.Invoke();
watch.Stop();
LogTask.Warn($"Stopping: {name} -- Time elapsed: {watch.Elapsed.TotalSeconds} seconds.");
return(watch.Elapsed);
}
public void TestLoadProcessKeyForLogTask()
{
StartLoadProcessTask.Start("Test process 8");
int?processKey1 = ControlFlow.CurrentLoadProcess.LoadProcessKey;
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
Assert.AreEqual(3, new SqlTask("Check if load process key is set", $"select count(*) from etl.Log where Message like 'Test%' and LoadProcessKey = {processKey1}")
{
DisableLogging = true
}.ExecuteScalar <int>());
}
public void TestDuplicateCheckInRowTrans()
{
CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv");
source.Configuration.Delimiter = ";";
source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim;
source.Configuration.MissingFieldFound = null;
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input =>
{
if (IDs.Contains(input.ID))
{
input.IsDuplicate = true;
}
else
{
IDs.Add(input.ID);
}
return(input);
});
var multicast = new Multicast <Poco>();
var dest = new DBDestination <Poco>("dbo.Staging");
TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>()
{
new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("ID", "INT", allowNulls: false),
new TableColumn("Value", "NVARCHAR(100)", allowNulls: false),
new TableColumn("Name", "NVARCHAR(100)", allowNulls: false)
});
stagingTable.CreateTable();
var trash = new CustomDestination <Poco>(input => {
LogTask.Warn($"Duplicate found. ID: {input.ID} Name: {input.Name}");
});
source.LinkTo(rowTrans);
rowTrans.LinkTo(multicast);
multicast.LinkTo(dest, input => input.IsDuplicate == false);
multicast.LinkTo(trash, input => input.IsDuplicate == true);
source.Execute();
dest.Wait();
trash.Wait();
}
public void LogCleanup()
{
//Arrange
LogTask.Error("Error");
LogTask.Warn("Warn");
LogTask.Info("Info");
//Act
CleanUpLogTask.CleanUp(Connection, 0);
//Assert
Assert.Equal(0, new RowCountTask("etl.Log ")
{
DisableLogging = true,
ConnectionManager = Connection
}.Count().Rows);
}
public void IsLoadProcessKeySetForLogTask()
{
//Arrange
StartLoadProcessTask.Start("Test process 15");
int?processKey1 = ControlFlow.CurrentLoadProcess.LoadProcessKey;
//Act
LogTask.Error("Test1");
LogTask.Warn("Test2");
LogTask.Info("Test3");
//Assert
Assert.Equal(3, new RowCountTask("etl.Log",
$"Message like 'Test%' AND LoadProcessKey = {processKey1}")
{
DisableLogging = true
}.Count().Rows);
}
private void RunSubSequence()
{
Sequence.Execute("Test sub sequence 1.1", () =>
{
SqlTask.ExecuteNonQuery($"Sql #2", "Select 1 as test");
SqlTask.ExecuteNonQuery($"Sql #3", "Select 1 as test");
LogTask.Warn("Warn message #1");
});
Sequence.Execute("Test sub sequence 1.2", () =>
{
SqlTask.ExecuteNonQuery($"Sql #4", "Select 1 as test");
});
Sequence.Execute("Test sub sequence 1.3",
() =>
{
Sequence.Execute("Test sub sequence 2.1", () =>
{
Sequence.Execute("Test sub sequence 3.1", () =>
{
SqlTask.ExecuteNonQuery($"Sql #5", "Select 1 as test");
SqlTask.ExecuteNonQuery($"Sql #6", "Select 1 as test");
LogTask.Warn("Warn message #2");
});
CustomTask.Execute($"Custom #1", () => {; });
SqlTask.ExecuteNonQuery($"Sql #7", "Select 1 as test");
});
Sequence.Execute("Test sub sequence 2.2", () =>
{
CustomTask.Execute($"Custom #2", () => {; });
SqlTask.ExecuteNonQuery($"Sql #7", "Select 1 as test");
});
Sequence.Execute("Test sub sequence 2.3", () =>
{
SqlTask.ExecuteNonQuery($"Sql #8", "Select 1 as test");
CustomTask.Execute($"Custom #2", () => {; });
Sequence.Execute("Test sub sequence 3.3", () =>
{
SqlTask.ExecuteNonQuery($"Sql #9", "Select 1 as test");
SqlTask.ExecuteNonQuery($"Sql #10", "Select 1 as test");
LogTask.Error("Error message");
});
});
});
CustomTask.Execute($"Custom #3", () => {; });
}