private static EtlSessionArguments GetArguments(CompiledModule module, string instance)
{
var argumentValues = new Dictionary <string, object>(StringComparer.InvariantCultureIgnoreCase);
foreach (var provider in module.DefaultArgumentProviders)
{
var values = provider.Arguments;
if (values != null)
{
foreach (var kvp in values)
{
argumentValues[kvp.Key] = kvp.Value;
}
}
}
foreach (var provider in module.InstanceArgumentProviders.Where(x => string.Equals(x.Instance, instance, StringComparison.InvariantCultureIgnoreCase)))
{
var values = provider.Arguments;
if (values != null)
{
foreach (var kvp in values)
{
argumentValues[kvp.Key] = kvp.Value;
}
}
}
return(new EtlSessionArguments(argumentValues));
}
public static void UnloadModule(CommandContext commandContext, CompiledModule module)
{
commandContext.Logger.Debug("unloading module {Module}", module.Name);
module.TaskTypes.Clear();
module.LoadContext?.Unload();
}
public static ExecutionResult LoadModule(CommandContext commandContext, string moduleName, bool forceCompilation, out CompiledModule module)
{
module = null;
var moduleFolder = Path.Combine(commandContext.HostConfiguration.ModulesFolder, moduleName);
if (!Directory.Exists(moduleFolder))
{
commandContext.Logger.Write(LogEventLevel.Fatal, "can't find the module folder: {Folder}", moduleFolder);
return(ExecutionResult.ModuleLoadError);
}
// read back actual folder name casing
moduleFolder = Directory
.GetDirectories(commandContext.HostConfiguration.ModulesFolder, moduleName, SearchOption.TopDirectoryOnly)
.FirstOrDefault();
moduleName = Path.GetFileName(moduleFolder);
var startedOn = Stopwatch.StartNew();
var useAppDomain = !forceCompilation && Debugger.IsAttached;
if (useAppDomain)
{
commandContext.Logger.Information("loading module directly from AppDomain where namespace ends with '{Module}'", moduleName);
var appDomainTasks = FindTypesFromAppDomain <IEtlTask>(moduleName);
var startup = LoadInstancesFromAppDomain <IStartup>(moduleName).FirstOrDefault();
var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName);
var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName);
commandContext.Logger.Debug("finished in {Elapsed}", startedOn.Elapsed);
module = new CompiledModule()
{
Name = moduleName,
Folder = moduleFolder,
Startup = startup,
InstanceArgumentProviders = instanceConfigurationProviders,
DefaultArgumentProviders = defaultConfigurationProviders,
TaskTypes = appDomainTasks.Where(x => x.Name != null).ToList(),
LoadContext = null,
};
commandContext.Logger.Debug("{FlowCount} flows(s) found: {Task}",
module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
commandContext.Logger.Debug("{TaskCount} task(s) found: {Task}",
module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
return(ExecutionResult.Success);
}
commandContext.Logger.Information("compiling module from {Folder}", PathHelpers.GetFriendlyPathName(moduleFolder));
var selfFolder = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
var referenceAssemblyFolder = commandContext.HostConfiguration.CustomReferenceAssemblyFolder;
if (string.IsNullOrEmpty(referenceAssemblyFolder))
{
referenceAssemblyFolder = Directory.GetDirectories(@"c:\Program Files\dotnet\shared\Microsoft.NETCore.App\", "6.*")
.OrderByDescending(x => new DirectoryInfo(x).CreationTime)
.FirstOrDefault();
}
commandContext.Logger.Information("using assemblies from {ReferenceAssemblyFolder}", referenceAssemblyFolder);
var referenceDllFileNames = new List <string>();
referenceDllFileNames.AddRange(Directory.GetFiles(referenceAssemblyFolder, "System*.dll", SearchOption.TopDirectoryOnly));
referenceDllFileNames.AddRange(Directory.GetFiles(referenceAssemblyFolder, "netstandard.dll", SearchOption.TopDirectoryOnly));
var referenceFileNames = new List <string>();
referenceFileNames.AddRange(referenceDllFileNames.Where(x => !Path.GetFileNameWithoutExtension(x).EndsWith("Native", StringComparison.InvariantCultureIgnoreCase)));
var localDllFileNames = Directory.GetFiles(selfFolder, "*.dll", SearchOption.TopDirectoryOnly)
.Where(x => Path.GetFileName(x) != "FizzCode.EtLast.ConsoleHost.dll" &&
!Path.GetFileName(x).Equals("testhost.dll", StringComparison.InvariantCultureIgnoreCase));
referenceFileNames.AddRange(localDllFileNames);
/*foreach (var fn in referenceFileNames)
* Console.WriteLine(fn);*/
var metadataReferences = referenceFileNames
.Distinct()
.Select(fn => MetadataReference.CreateFromFile(fn))
.ToArray();
var csFileNames = Directory.GetFiles(moduleFolder, "*.cs", SearchOption.AllDirectories);
var parseOptions = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.CSharp10);
var syntaxTrees = csFileNames
.Select(fn => SyntaxFactory.ParseSyntaxTree(SourceText.From(File.ReadAllText(fn)), parseOptions, fn))
.ToArray();
using (var assemblyStream = new MemoryStream())
{
var id = Interlocked.Increment(ref _moduleAutoincrementId);
var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary,
optimizationLevel: OptimizationLevel.Release,
assemblyIdentityComparer: DesktopAssemblyIdentityComparer.Default);
var compilation = CSharpCompilation.Create("compiled_" + id.ToString("D", CultureInfo.InvariantCulture) + ".dll", syntaxTrees, metadataReferences, compilationOptions);
var result = compilation.Emit(assemblyStream);
if (!result.Success)
{
var failures = result.Diagnostics.Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error);
foreach (var error in failures)
{
commandContext.Logger.Write(LogEventLevel.Fatal, "syntax error in module: {ErrorMessage}", error.ToString());
}
return(ExecutionResult.ModuleLoadError);
}
assemblyStream.Seek(0, SeekOrigin.Begin);
var assemblyLoadContext = new AssemblyLoadContext(null, isCollectible: true);
var assembly = assemblyLoadContext.LoadFromStream(assemblyStream);
var compiledTasks = FindTypesFromAssembly <IEtlTask>(assembly);
var compiledStartup = LoadInstancesFromAssembly <IStartup>(assembly).FirstOrDefault();
var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName);
var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName);
commandContext.Logger.Debug("compilation finished in {Elapsed}", startedOn.Elapsed);
module = new CompiledModule()
{
Name = moduleName,
Folder = moduleFolder,
Startup = compiledStartup,
InstanceArgumentProviders = instanceConfigurationProviders,
DefaultArgumentProviders = defaultConfigurationProviders,
TaskTypes = compiledTasks.Where(x => x.Name != null).ToList(),
LoadContext = assemblyLoadContext,
};
commandContext.Logger.Debug("{FlowCount} flows(s) found: {Task}",
module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
commandContext.Logger.Debug("{TaskCount} task(s) found: {Task}",
module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
return(ExecutionResult.Success);
}
}
public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands)
{
var result = ExecutionResult.Success;
var etlContext = new EtlContext();
try
{
var listeners = commandContext.HostConfiguration.GetSessionListeners(null);
if (listeners?.Count > 0)
{
etlContext.Listeners.AddRange(listeners);
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
var instance = Environment.MachineName;
var arguments = GetArguments(module, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, etlContext, arguments);
etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
var logger = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var ioLogger = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder);
var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder);
etlContext.Listeners.Add(serilogAdapter);
serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptionCount = 0;
var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >();
try
{
foreach (var command in commands)
{
IEtlTask task = null;
if (taskCreators.TryGetValue(command, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task);
taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult));
sessionExceptionCount += taskResult.ExceptionCount;
if (sessionExceptionCount > 0)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution");
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message);
result = ExecutionResult.ExecutionFailed;
etlContext.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(serilogAdapter, task);
}
session.Stop();
if (taskResults.Count > 0)
{
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY");
serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------");
var longestTaskName = taskResults.Max(x => x.Key.Name.Length);
foreach (var kvp in taskResults)
{
LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName);
}
}
etlContext.Close();
}
catch (TransactionAbortedException)
{
}
return(result);
}