public void LogEtlSessionEnd(EtlSession session)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
var cmdText = string.Format(@"
update {0}[EtlSessions]
set
[EndDateTime] = getdate(),
[EndUtcDateTime] = getutcdate(),
[Status] = @Status
where
[EtlPackageId] = @EtlPackageId and
[EtlSessionId] = @EtlSessionId
", _schemaToken);
using (var conn = CreateConnection())
{
conn.Open();
var cmd = new SqlCommand(cmdText, conn);
AddParameter(cmd, "@EtlPackageId", session.EtlPackageId);
AddParameter(cmd, "@EtlSessionId", session.EtlSessionId);
AddParameter(cmd, "@EndDateTime", session.EndDateTime);
AddParameter(cmd, "@EndUtcDateTime", session.EndUtcDateTime);
AddParameter(cmd, "@Status", (int)session.Status);
cmd.ExecuteNonQuery();
}
}
private static EtlSessionSummary GetEtlSessionSummary(EtlSession session, IEtlLogParser logParser)
{
var sessionSummary = new EtlSessionSummary
{
EtlPackageId = session.EtlPackageId,
EtlPackageName = session.EtlPackageName,
EtlSessionId = session.EtlSessionId,
StartDateTime = session.StartDateTime,
StartUtcDateTime = session.StartUtcDateTime,
EndDateTime = session.EndDateTime,
EndUtcDateTime = session.EndUtcDateTime,
Status = session.Status,
UserName = session.UserName,
};
var variables = logParser.GetEtlVariables(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Variables.AddRange(variables);
var counters = logParser.GetEtlCounters(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Counters.AddRange(counters);
var messages = logParser.GetEtlMessages(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Messages.AddRange(messages);
for (var i = messages.Length - 1; i >= 0; i--)
{
if (messages[i].MessageType == EtlMessageType.Error)
{
sessionSummary.LastErrorMessage = messages[i];
break;
}
}
return sessionSummary;
}
public void LogEtlSessionEnd(EtlSession session)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
_currentProgress = 100;
//_worker.ReportProgress(_currentProgress, null);
}
public static EtlPackageMonitorItemStatus GetMonitorItemStatus(EtlSession session, int runInvervalSeconds)
{
return(session.Status ==
EtlStatus.Failed ? EtlPackageMonitorItemStatus.Failed :
// пакет выполнялся слишком давно
session.EndDateTime.HasValue?
runInvervalSeconds > 0 && DateTime.Now.Subtract(session.EndDateTime.Value).TotalSeconds >= runInvervalSeconds ?
EtlPackageMonitorItemStatus.TooFar :
session.Status == EtlStatus.Succeeded ?
EtlPackageMonitorItemStatus.Succeeded :
EtlPackageMonitorItemStatus.Failed:
EtlPackageMonitorItemStatus.Running);
}
public void LogEtlSessionStart(EtlSession session)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
var insertSessionSql = string.Format(@"
insert into {0}[EtlSessions]
(
[EtlPackageId],
[EtlPackageName],
[EtlSessionId],
[StartDateTime],
[StartUtcDateTime],
[Status],
[ParentEtlSessionId],
[UserName]
)
values
(
@EtlPackageId,
@EtlPackageName,
@EtlSessionId,
@StartDateTime,
@StartUtcDateTime,
@Status,
@ParentEtlSessionId,
@userName
)
", _schemaToken);
var insertSessionCommand = new SqlCommand(insertSessionSql);
AddParameter(insertSessionCommand, "@EtlPackageId", session.EtlPackageId);
AddParameter(insertSessionCommand, "@EtlPackageName", session.EtlPackageName);
AddParameter(insertSessionCommand, "@EtlSessionId", session.EtlSessionId);
AddParameter(insertSessionCommand, "@StartDateTime", session.StartDateTime);
AddParameter(insertSessionCommand, "@StartUtcDateTime", session.StartUtcDateTime);
AddParameter(insertSessionCommand, "@Status", (int)EtlStatus.Started);
AddParameter(insertSessionCommand, "@ParentEtlSessionId", session.ParentEtlSessionId);
AddParameter(insertSessionCommand, "@UserName", GetSubstring(session.UserName, EtlSessionUserNameMaxLength));
using (var conn = CreateConnection())
{
conn.Open();
insertSessionCommand.Connection = conn;
insertSessionCommand.ExecuteNonQuery();
}
}
public EtlSession GetEtlSession(string etlPackageId, string etlSessionId)
{
EtlSession result = null;
foreach (var agent in _etlAgents)
{
var logParser = agent.GetEtlLogParser();
result = logParser.GetEtlSession(etlPackageId, etlSessionId);
if (result != null)
{
break;
}
}
return(result);
}
public EtlContext
(
EtlSession session
)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
_parentSessionId = session.ParentEtlSessionId;
_packageId = session.EtlPackageId;
_sessionId = session.EtlSessionId;
_userName = session.UserName;
_startDateTime = session.StartDateTime;
_startUtcDateTime = session.StartUtcDateTime;
}
private EtlSession ReadEtlSession(SqlDataReader reader)
{
var session = new EtlSession
{
EtlPackageId = EtlValueConverter.ToString(reader["EtlPackageId"]),
EtlPackageName = EtlValueConverter.ToString(reader["EtlPackageName"]),
EtlSessionId = EtlValueConverter.ToString(reader["EtlSessionId"]),
StartDateTime = EtlValueConverter.ParseDateTime(reader["StartDateTime"]),
StartUtcDateTime = EtlValueConverter.ParseDateTime(reader["StartUtcDateTime"]),
EndDateTime = EtlValueConverter.ParseDateTimeOrNull(reader["EndDateTime"]),
EndUtcDateTime = EtlValueConverter.ParseDateTimeOrNull(reader["EndUtcDateTime"]),
Status = (EtlStatus)EtlValueConverter.ParseInt32(reader["Status"]),
UserName = EtlValueConverter.ToString(reader["UserName"]),
};
return(session);
}
private static EtlSessionSummary GetEtlSessionSummary(EtlSession session, IEtlLogParser logParser)
{
var sessionSummary = new EtlSessionSummary
{
EtlPackageId = session.EtlPackageId,
EtlPackageName = session.EtlPackageName,
EtlSessionId = session.EtlSessionId,
StartDateTime = session.StartDateTime,
StartUtcDateTime = session.StartUtcDateTime,
EndDateTime = session.EndDateTime,
EndUtcDateTime = session.EndUtcDateTime,
Status = session.Status,
UserName = session.UserName,
};
var variables = logParser.GetEtlVariables(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Variables.AddRange(variables);
var counters = logParser.GetEtlCounters(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Counters.AddRange(counters);
var messages = logParser.GetEtlMessages(session.EtlPackageId, session.EtlSessionId);
sessionSummary.Messages.AddRange(messages);
for (var i = messages.Length - 1; i >= 0; i--)
{
if (messages[i].MessageType == EtlMessageType.Error)
{
sessionSummary.LastErrorMessage = messages[i];
break;
}
}
return(sessionSummary);
}
public static IExecutionResult Execute(Host host, CompiledModule module, string[] taskNames)
{
var executionResult = new ExecutionResult
{
TaskResults = new List <TaskExectionResult>(),
};
var instance = Environment.MachineName;
var arguments = new ArgumentCollection(module.DefaultArgumentProviders, module.InstanceArgumentProviders, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var customTasks = new Dictionary <string, Func <IArgumentCollection, IEtlTask> >(module.Startup.CustomTasks, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, arguments);
session.Context.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
try
{
if (host.EtlContextListeners?.Count > 0)
{
foreach (var listenerCreator in host.EtlContextListeners)
{
session.Context.Listeners.Add(listenerCreator.Invoke(session));
}
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
session.Context.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
session.Context.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
if (host.SerilogForModulesEnabled)
{
if (environmentSettings.FileLogSettings.Enabled || environmentSettings.ConsoleLogSettings.Enabled || !string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
var serilogAdapter = new EtlSessionSerilogAdapter(environmentSettings, host.DevLogFolder, host.OpsLogFolder);
session.Context.Listeners.Add(serilogAdapter);
}
}
session.Context.Log(LogSeverity.Information, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
session.Context.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptions = new List <Exception>();
var taskResults = new List <TaskExectionResult>();
try
{
foreach (var taskName in taskNames)
{
IEtlTask task = null;
if (customTasks.TryGetValue(taskName, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, taskName, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
session.Context.Log(LogSeverity.Error, null, "unknown task/flow type: " + taskName);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task) as ProcessResult;
taskResults.Add(new TaskExectionResult(task, taskResult));
executionResult.TaskResults.Add(new TaskExectionResult(task, taskResult));
sessionExceptions.AddRange(taskResult.Exceptions);
if (sessionExceptions.Count > 0)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution");
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution, reason: {0}", ex.Message);
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(session.Context, task);
}
session.Stop();
if (taskResults.Count > 0)
{
session.Context.Log(LogSeverity.Information, null, "-------");
session.Context.Log(LogSeverity.Information, null, "SUMMARY");
session.Context.Log(LogSeverity.Information, null, "-------");
var longestTaskName = taskResults.Max(x => x.TaskName.Length);
foreach (var taskResult in taskResults)
{
LogTaskSummary(session.Context, taskResult, longestTaskName);
}
}
session.Context.Close();
}
catch (TransactionAbortedException)
{
}
return(executionResult);
}
public void LogEtlSessionEnd(EtlSession session)
{
}
private void InvokePackageCatched(EtlPackage package, EtlSession session, EtlVariableAssignment[] assignments)
{
var buildStartDateTime = DateTime.Now;
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = buildStartDateTime,
LogUtcDateTime = buildStartDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Debug,
Text = string.Format(Properties.Resources.VariablesInitStarted),
}
);
var context = new _Context(package, session);
context.InitVariables(package.Variables, assignments);
foreach (var variable in context.GetVariables())
{
var escapedVariable = EscapeVariable(variable);
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = context.EtlPackageId,
EtlSessionId = context.EtlSessionId,
LogDateTime = buildStartDateTime,
LogUtcDateTime = buildStartDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Debug,
Text = string.Format(Properties.Resources.VariableInit, escapedVariable.Name, escapedVariable.Value),
}
);
if (variable.Modifier == EtlVariableModifier.Input || variable.Modifier == EtlVariableModifier.Bound)
{
_logger.LogEtlVariable(escapedVariable);
}
}
var preprocessor = new EtlPackagePreprocessor();
package = preprocessor.PreprocessPackage(package, context.GetVariables());
var buildEndDateTime = DateTime.Now;
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = buildEndDateTime,
LogUtcDateTime = buildEndDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Debug,
Text = string.Format(Properties.Resources.VariablesInitFinished),
}
);
InvokePackageSteps(package, context);
foreach (var variable in context.GetVariables())
{
if (variable.Modifier == EtlVariableModifier.Output)
{
_logger.LogEtlVariable(EscapeVariable(variable));
}
}
session.Status = context.CurrentStatus;
}
public void LogEtlSessionStart(EtlSession session)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
var insertSessionSql = string.Format(@"
insert into {0}[EtlSessions]
(
[EtlPackageId],
[EtlPackageName],
[EtlSessionId],
[StartDateTime],
[StartUtcDateTime],
[Status],
[ParentEtlSessionId],
[UserName]
)
values
(
@EtlPackageId,
@EtlPackageName,
@EtlSessionId,
@StartDateTime,
@StartUtcDateTime,
@Status,
@ParentEtlSessionId,
@userName
)
", _schemaToken);
var insertSessionCommand = new SqlCommand(insertSessionSql);
AddParameter(insertSessionCommand, "@EtlPackageId", session.EtlPackageId);
AddParameter(insertSessionCommand, "@EtlPackageName", session.EtlPackageName);
AddParameter(insertSessionCommand, "@EtlSessionId", session.EtlSessionId);
AddParameter(insertSessionCommand, "@StartDateTime", session.StartDateTime);
AddParameter(insertSessionCommand, "@StartUtcDateTime", session.StartUtcDateTime);
AddParameter(insertSessionCommand, "@Status", (int)EtlStatus.Started);
AddParameter(insertSessionCommand, "@ParentEtlSessionId", session.ParentEtlSessionId);
AddParameter(insertSessionCommand, "@UserName", GetSubstring(session.UserName, EtlSessionUserNameMaxLength));
using (var conn = CreateConnection())
{
conn.Open();
insertSessionCommand.Connection = conn;
insertSessionCommand.ExecuteNonQuery();
}
}
public void LogEtlSessionEnd(EtlSession session)
{
if (session == null)
{
throw new ArgumentNullException("session");
}
var cmdText = string.Format(@"
update {0}[EtlSessions]
set
[EndDateTime] = getdate(),
[EndUtcDateTime] = getutcdate(),
[Status] = @Status
where
[EtlPackageId] = @EtlPackageId and
[EtlSessionId] = @EtlSessionId
", _schemaToken);
using (var conn = CreateConnection())
{
conn.Open();
var cmd = new SqlCommand(cmdText, conn);
AddParameter(cmd, "@EtlPackageId", session.EtlPackageId);
AddParameter(cmd, "@EtlSessionId", session.EtlSessionId);
AddParameter(cmd, "@EndDateTime", session.EndDateTime);
AddParameter(cmd, "@EndUtcDateTime", session.EndUtcDateTime);
AddParameter(cmd, "@Status", (int)session.Status);
cmd.ExecuteNonQuery();
}
}
public _Context(EtlPackage originalPackage, EtlSession session)
: base(session)
{
_currentStatus = session.Status;
_originalPackage = originalPackage;
}
private EtlSession InvokePackageCore(EtlPackage package, EtlVariableAssignment[] assignments, string parentSessionId)
{
var session = new EtlSession
{
EtlSessionId = Guid.NewGuid().ToString(),
EtlPackageId = package.Id,
EtlPackageName = package.Name,
ParentEtlSessionId = parentSessionId,
Status = EtlStatus.Started,
UserName = GetCurrentUserName(),
};
session.StartDateTime = DateTime.Now;
session.StartUtcDateTime = session.StartDateTime.ToUniversalTime();
_logger.LogEtlSessionStart(session);
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = session.StartDateTime,
LogUtcDateTime = session.StartUtcDateTime,
MessageType = EtlMessageType.SessionStart,
Text = string.Format(Properties.Resources.SessionStarted, session.EtlSessionId, session.EtlPackageId, session.EtlPackageName)
}
);
try
{
InvokePackageCatched(package, session, assignments);
}
catch (Exception exc)
{
if (IsCriticalException(exc))
{
throw;
}
session.Status = EtlStatus.Failed;
var errorDateTime = DateTime.Now;
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = errorDateTime,
LogUtcDateTime = errorDateTime.ToUniversalTime(),
MessageType = EtlMessageType.Error,
Text = exc.Message,
StackTrace = exc.StackTrace,
}
);
}
finally
{
session.EndDateTime = DateTime.Now;
session.EndUtcDateTime = session.EndDateTime.Value.ToUniversalTime();
_logger.LogEtlMessage
(
new EtlMessage
{
EtlPackageId = session.EtlPackageId,
EtlSessionId = session.EtlSessionId,
LogDateTime = session.EndDateTime.Value,
LogUtcDateTime = session.EndUtcDateTime.Value,
MessageType = EtlMessageType.SessionEnd,
Text = string.Format(Properties.Resources.SessionFinished, session.EtlSessionId, session.EtlPackageId, session.EtlPackageName, session.Status)
}
);
_logger.LogEtlSessionEnd(session);
}
return(session);
}
public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
{
var result = ExecutionResult.Success;
var etlContext = new EtlContext();
try
{
var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
if (listeners?.Count > 0)
{
etlContext.Listeners.AddRange(listeners);
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
var instance = Environment.MachineName;
var arguments = GetArguments(module, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, etlContext, arguments);
etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
var logger = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var ioLogger = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);
etlContext.Listeners.Add(serilogAdapter);
serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptionCount = 0;
var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();
try
{
foreach (var command in commands)
{
IEtlTask task = null;
if (taskCreators.TryGetValue(command, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task);
taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));
sessionExceptionCount += taskResult.ExceptionCount;
if (sessionExceptionCount > 0)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(serilogAdapter, task);
}
session.Stop();
if (taskResults.Count > 0)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
foreach (var kvp in taskResults)
{
LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
}
}
etlContext.Close();
}
catch (TransactionAbortedException)
{
}
return(result);
}
public void LogEtlSessionStart(EtlSession session)
{
}
private EtlSession ReadEtlSession(SqlDataReader reader)
{
var session = new EtlSession
{
EtlPackageId = EtlValueConverter.ToString(reader["EtlPackageId"]),
EtlPackageName = EtlValueConverter.ToString(reader["EtlPackageName"]),
EtlSessionId = EtlValueConverter.ToString(reader["EtlSessionId"]),
StartDateTime = EtlValueConverter.ParseDateTime(reader["StartDateTime"]),
StartUtcDateTime = EtlValueConverter.ParseDateTime(reader["StartUtcDateTime"]),
EndDateTime = EtlValueConverter.ParseDateTimeOrNull(reader["EndDateTime"]),
EndUtcDateTime = EtlValueConverter.ParseDateTimeOrNull(reader["EndUtcDateTime"]),
Status = (EtlStatus)EtlValueConverter.ParseInt32(reader["Status"]),
UserName = EtlValueConverter.ToString(reader["UserName"]),
};
return session;
}
