Exemplo n.º 1
0
 public TaskExectionResult(IEtlTask task, ProcessResult result)
 {
     TaskType          = task.GetType();
     TaskName          = task.Name;
     TaskKind          = task.Kind;
     TaskTopic         = task.GetTopic();
     Statistics        = task.Statistics;
     IoCommandCounters = task.IoCommandCounters;
     Exceptions        = result.Exceptions.ToList();
 }
Exemplo n.º 2
0
        private static string GetProcessKind(IProcess process)
        {
            if (process.GetType().GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IExecutableWithResult <>)))
            {
                return("jobWithResult");
            }

            return(process switch
            {
                IEtlFlow _ => "flow",
                IEtlTask _ => "task",
                IRowSource _ => "source",
                IRowSink _ => "sink",
                IMutator _ => "mutator",
                IScope _ => "scope",
                IProducer _ => "producer",
                IExecutable _ => "job",
                _ => "unknown",
            });
Exemplo n.º 3
0
        private static void LogTaskSummary(EtlSessionSerilogAdapter serilogAdapter, IEtlTask task, TaskResult result, int longestTaskName)
        {
            var spacing1            = "".PadRight(longestTaskName - task.Name.Length);
            var spacing1WithoutName = "".PadRight(longestTaskName);

            if (result.ExceptionCount == 0)
            {
                serilogAdapter.Log(LogSeverity.Information, false, null, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
                                   task.Name, spacing1, task.Statistics.RunTime, "success", task.Statistics.CpuTime, task.Statistics.TotalAllocations, task.Statistics.AllocationDifference);
            }
            else
            {
                serilogAdapter.Log(LogSeverity.Information, false, null, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
                                   task.Name, spacing1, task.Statistics.RunTime, "failed", task.Statistics.CpuTime, task.Statistics.TotalAllocations, task.Statistics.AllocationDifference);
            }

            if (task.IoCommandCounters.Count > 0)
            {
                var maxKeyLength        = task.IoCommandCounters.Max(x => x.Key.ToString().Length);
                var maxInvocationLength = task.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length);

                foreach (var kvp in task.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
                {
                    if (kvp.Value.AffectedDataCount != null)
                    {
                        serilogAdapter.Log(LogSeverity.Information, false, null, null, "{spacing1} {Kind}{spacing2} {InvocationCount}{spacing3}   {AffectedDataCount}", spacing1WithoutName,
                                           kvp.Key.ToString(),
                                           "".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
                                           kvp.Value.InvocationCount,
                                           "".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
                                           kvp.Value.AffectedDataCount);
                    }
                    else
                    {
                        serilogAdapter.Log(LogSeverity.Information, false, null, null, "{spacing1} {Kind}{spacing2} {InvocationCount}", spacing1WithoutName,
                                           kvp.Key.ToString(),
                                           "".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
                                           kvp.Value.InvocationCount);
                    }
                }
            }
        }
Exemplo n.º 4
0
        private static void LogTaskCounters(EtlSessionSerilogAdapter serilogAdapter, IEtlTask task)
        {
            if (task.IoCommandCounters.Count == 0)
            {
                return;
            }

            const string kind       = "kind";
            const string invocation = "invoc.";
            const string affected   = "affected";

            var maxKeyLength        = Math.Max(kind.Length, task.IoCommandCounters.Max(x => x.Key.ToString().Length));
            var maxInvocationLength = Math.Max(invocation.Length, task.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length));

            serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}{spacing2}   {AffectedDataCount}", kind,
                               "".PadRight(maxKeyLength - kind.Length, ' '),
                               invocation,
                               "".PadRight(maxInvocationLength - invocation.Length, ' '),
                               affected);

            foreach (var kvp in task.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
            {
                if (kvp.Value.AffectedDataCount != null)
                {
                    serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}{spacing2}   {AffectedDataCount}", kvp.Key.ToString(),
                                       "".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
                                       kvp.Value.InvocationCount,
                                       "".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
                                       kvp.Value.AffectedDataCount);
                }
                else
                {
                    serilogAdapter.Log(LogSeverity.Debug, false, null, null, "{Kind}{spacing1} {InvocationCount}", kvp.Key.ToString(),
                                       "".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '), kvp.Value.InvocationCount);
                }
            }
        }
Exemplo n.º 5
0
        public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
        {
            var result = ExecutionResult.Success;

            var etlContext = new EtlContext();

            try
            {
                var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
                if (listeners?.Count > 0)
                {
                    etlContext.Listeners.AddRange(listeners);
                }
            }
            catch (Exception ex)
            {
                var formattedMessage = ex.FormatExceptionWithDetails();
                etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
                etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
            }

            var instance  = Environment.MachineName;
            var arguments = GetArguments(module, instance);

            var environmentSettings = new EnvironmentSettings();

            module.Startup.Configure(environmentSettings);
            var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);

            var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
            var session   = new EtlSession(sessionId, etlContext, arguments);

            etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;

            var logger    = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
            var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
            var ioLogger  = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);

            var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);

            etlContext.Listeners.Add(serilogAdapter);

            serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);

            if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
            {
                etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
            }

            var sessionStartedOn      = Stopwatch.StartNew();
            var sessionExceptionCount = 0;

            var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();

            try
            {
                foreach (var command in commands)
                {
                    IEtlTask task = null;
                    if (taskCreators.TryGetValue(command, out var taskCreator))
                    {
                        task = taskCreator.Invoke(arguments);
                    }
                    else
                    {
                        var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
                        if (taskType != null)
                        {
                            task = (IEtlTask)Activator.CreateInstance(taskType);
                        }
                    }

                    if (task == null)
                    {
                        serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
                        break;
                    }

                    try
                    {
                        try
                        {
                            var taskResult = session.ExecuteTask(null, task);
                            taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));

                            sessionExceptionCount += taskResult.ExceptionCount;

                            if (sessionExceptionCount > 0)
                            {
                                etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
                                result = ExecutionResult.ExecutionFailed;
                                etlContext.Close();
                                break; // stop processing tasks
                            }
                        }
                        catch (Exception ex)
                        {
                            etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
                            result = ExecutionResult.ExecutionFailed;
                            etlContext.Close();
                            break; // stop processing tasks
                        }
                    }
                    catch (TransactionAbortedException)
                    {
                    }

                    LogTaskCounters(serilogAdapter, task);
                }

                session.Stop();

                if (taskResults.Count > 0)
                {
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
                    serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");

                    var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
                    foreach (var kvp in taskResults)
                    {
                        LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
                    }
                }

                etlContext.Close();
            }
            catch (TransactionAbortedException)
            {
            }

            return(result);
        }
Exemplo n.º 6
0
    public static IExecutionResult Execute(Host host, CompiledModule module, string[] taskNames)
    {
        var executionResult = new ExecutionResult
        {
            TaskResults = new List <TaskExectionResult>(),
        };

        var instance  = Environment.MachineName;
        var arguments = new ArgumentCollection(module.DefaultArgumentProviders, module.InstanceArgumentProviders, instance);

        var environmentSettings = new EnvironmentSettings();

        module.Startup.Configure(environmentSettings);
        var customTasks = new Dictionary <string, Func <IArgumentCollection, IEtlTask> >(module.Startup.CustomTasks, StringComparer.InvariantCultureIgnoreCase);

        var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
        var session   = new EtlSession(sessionId, arguments);

        session.Context.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;

        try
        {
            if (host.EtlContextListeners?.Count > 0)
            {
                foreach (var listenerCreator in host.EtlContextListeners)
                {
                    session.Context.Listeners.Add(listenerCreator.Invoke(session));
                }
            }
        }
        catch (Exception ex)
        {
            var formattedMessage = ex.FormatExceptionWithDetails();
            session.Context.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
            session.Context.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
        }

        if (host.SerilogForModulesEnabled)
        {
            if (environmentSettings.FileLogSettings.Enabled || environmentSettings.ConsoleLogSettings.Enabled || !string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
            {
                var serilogAdapter = new EtlSessionSerilogAdapter(environmentSettings, host.DevLogFolder, host.OpsLogFolder);
                session.Context.Listeners.Add(serilogAdapter);
            }
        }

        session.Context.Log(LogSeverity.Information, null, "session {SessionId} started", sessionId);

        if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
        {
            session.Context.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
        }

        var sessionStartedOn  = Stopwatch.StartNew();
        var sessionExceptions = new List <Exception>();

        var taskResults = new List <TaskExectionResult>();

        try
        {
            foreach (var taskName in taskNames)
            {
                IEtlTask task = null;
                if (customTasks.TryGetValue(taskName, out var taskCreator))
                {
                    task = taskCreator.Invoke(arguments);
                }
                else
                {
                    var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, taskName, StringComparison.InvariantCultureIgnoreCase));
                    if (taskType != null)
                    {
                        task = (IEtlTask)Activator.CreateInstance(taskType);
                    }
                }

                if (task == null)
                {
                    session.Context.Log(LogSeverity.Error, null, "unknown task/flow type: " + taskName);
                    break;
                }

                try
                {
                    try
                    {
                        var taskResult = session.ExecuteTask(null, task) as ProcessResult;
                        taskResults.Add(new TaskExectionResult(task, taskResult));
                        executionResult.TaskResults.Add(new TaskExectionResult(task, taskResult));

                        sessionExceptions.AddRange(taskResult.Exceptions);

                        if (sessionExceptions.Count > 0)
                        {
                            session.Context.Log(LogSeverity.Error, task, "failed, terminating execution");
                            executionResult.Status = ExecutionStatusCode.ExecutionFailed;
                            session.Context.Close();
                            break; // stop processing tasks
                        }
                    }
                    catch (Exception ex)
                    {
                        session.Context.Log(LogSeverity.Error, task, "failed, terminating execution, reason: {0}", ex.Message);
                        executionResult.Status = ExecutionStatusCode.ExecutionFailed;
                        session.Context.Close();
                        break; // stop processing tasks
                    }
                }
                catch (TransactionAbortedException)
                {
                }

                LogTaskCounters(session.Context, task);
            }

            session.Stop();

            if (taskResults.Count > 0)
            {
                session.Context.Log(LogSeverity.Information, null, "-------");
                session.Context.Log(LogSeverity.Information, null, "SUMMARY");
                session.Context.Log(LogSeverity.Information, null, "-------");

                var longestTaskName = taskResults.Max(x => x.TaskName.Length);
                foreach (var taskResult in taskResults)
                {
                    LogTaskSummary(session.Context, taskResult, longestTaskName);
                }
            }

            session.Context.Close();
        }
        catch (TransactionAbortedException)
        {
        }

        return(executionResult);
    }