public TaskExectionResult(IEtlTask task, ProcessResult result)
{
TaskType = task.GetType();
TaskName = task.Name;
TaskKind = task.Kind;
TaskTopic = task.GetTopic();
Statistics = task.Statistics;
IoCommandCounters = task.IoCommandCounters;
Exceptions = result.Exceptions.ToList();
}
private static string GetProcessKind(IProcess process)
{
if (process.GetType().GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IExecutableWithResult <>)))
{
return("jobWithResult");
}
return(process switch
{
IEtlFlow _ => "flow",
IEtlTask _ => "task",
IRowSource _ => "source",
IRowSink _ => "sink",
IMutator _ => "mutator",
IScope _ => "scope",
IProducer _ => "producer",
IExecutable _ => "job",
_ => "unknown",
});
private static void LogTaskSummary(EtlSessionSerilogAdapter serilogAdapter, IEtlTask task, TaskResult result, int longestTaskName)
{
var spacing1 = "".PadRight(longestTaskName - task.Name.Length);
var spacing1WithoutName = "".PadRight(longestTaskName);
if (result.ExceptionCount == 0)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
task.Name, spacing1, task.Statistics.RunTime, "success", task.Statistics.CpuTime, task.Statistics.TotalAllocations, task.Statistics.AllocationDifference);
}
else
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
task.Name, spacing1, task.Statistics.RunTime, "failed", task.Statistics.CpuTime, task.Statistics.TotalAllocations, task.Statistics.AllocationDifference);
}
if (task.IoCommandCounters.Count > 0)
{
var maxKeyLength = task.IoCommandCounters.Max(x => x.Key.ToString().Length);
var maxInvocationLength = task.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length);
foreach (var kvp in task.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
{
if (kvp.Value.AffectedDataCount != null)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "{spacing1} {Kind}{spacing2} {InvocationCount}{spacing3} {AffectedDataCount}", spacing1WithoutName,
kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount,
"".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
kvp.Value.AffectedDataCount);
}
else
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "{spacing1} {Kind}{spacing2} {InvocationCount}", spacing1WithoutName,
kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount);
}
}
}
}
private static void LogTaskCounters(EtlSessionSerilogAdapter serilogAdapter, IEtlTask task)
{
if (task.IoCommandCounters.Count == 0)
{
return;
}
const string kind = "kind";
const string invocation = "invoc.";
const string affected = "affected";
var maxKeyLength = Math.Max(kind.Length, task.IoCommandCounters.Max(x => x.Key.ToString().Length));
var maxInvocationLength = Math.Max(invocation.Length, task.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length));
serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}{spacing2} {AffectedDataCount}", kind,
"".PadRight(maxKeyLength - kind.Length, ' '),
invocation,
"".PadRight(maxInvocationLength - invocation.Length, ' '),
affected);
foreach (var kvp in task.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
{
if (kvp.Value.AffectedDataCount != null)
{
serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}{spacing2} {AffectedDataCount}", kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount,
"".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
kvp.Value.AffectedDataCount);
}
else
{
serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}", kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '), kvp.Value.InvocationCount);
}
}
}
public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
{
var result = ExecutionResult.Success;
var etlContext = new EtlContext();
try
{
var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
if (listeners?.Count > 0)
{
etlContext.Listeners.AddRange(listeners);
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
var instance = Environment.MachineName;
var arguments = GetArguments(module, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, etlContext, arguments);
etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
var logger = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var ioLogger = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);
etlContext.Listeners.Add(serilogAdapter);
serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptionCount = 0;
var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();
try
{
foreach (var command in commands)
{
IEtlTask task = null;
if (taskCreators.TryGetValue(command, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task);
taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));
sessionExceptionCount += taskResult.ExceptionCount;
if (sessionExceptionCount > 0)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(serilogAdapter, task);
}
session.Stop();
if (taskResults.Count > 0)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
foreach (var kvp in taskResults)
{
LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
}
}
etlContext.Close();
}
catch (TransactionAbortedException)
{
}
return(result);
}
public static IExecutionResult Execute(Host host, CompiledModule module, string[] taskNames)
{
var executionResult = new ExecutionResult
{
TaskResults = new List <TaskExectionResult>(),
};
var instance = Environment.MachineName;
var arguments = new ArgumentCollection(module.DefaultArgumentProviders, module.InstanceArgumentProviders, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var customTasks = new Dictionary <string, Func <IArgumentCollection, IEtlTask> >(module.Startup.CustomTasks, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, arguments);
session.Context.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
try
{
if (host.EtlContextListeners?.Count > 0)
{
foreach (var listenerCreator in host.EtlContextListeners)
{
session.Context.Listeners.Add(listenerCreator.Invoke(session));
}
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
session.Context.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
session.Context.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
if (host.SerilogForModulesEnabled)
{
if (environmentSettings.FileLogSettings.Enabled || environmentSettings.ConsoleLogSettings.Enabled || !string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
var serilogAdapter = new EtlSessionSerilogAdapter(environmentSettings, host.DevLogFolder, host.OpsLogFolder);
session.Context.Listeners.Add(serilogAdapter);
}
}
session.Context.Log(LogSeverity.Information, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
session.Context.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptions = new List <Exception>();
var taskResults = new List <TaskExectionResult>();
try
{
foreach (var taskName in taskNames)
{
IEtlTask task = null;
if (customTasks.TryGetValue(taskName, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, taskName, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
session.Context.Log(LogSeverity.Error, null, "unknown task/flow type: " + taskName);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task) as ProcessResult;
taskResults.Add(new TaskExectionResult(task, taskResult));
executionResult.TaskResults.Add(new TaskExectionResult(task, taskResult));
sessionExceptions.AddRange(taskResult.Exceptions);
if (sessionExceptions.Count > 0)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution");
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution, reason: {0}", ex.Message);
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(session.Context, task);
}
session.Stop();
if (taskResults.Count > 0)
{
session.Context.Log(LogSeverity.Information, null, "-------");
session.Context.Log(LogSeverity.Information, null, "SUMMARY");
session.Context.Log(LogSeverity.Information, null, "-------");
var longestTaskName = taskResults.Max(x => x.TaskName.Length);
foreach (var taskResult in taskResults)
{
LogTaskSummary(session.Context, taskResult, longestTaskName);
}
}
session.Context.Close();
}
catch (TransactionAbortedException)
{
}
return(executionResult);
}