private static EtlSessionArguments GetArguments(CompiledModule module, string instance) { var argumentValues = new Dictionary <string, object>(StringComparer.InvariantCultureIgnoreCase); foreach (var provider in module.DefaultArgumentProviders) { var values = provider.Arguments; if (values != null) { foreach (var kvp in values) { argumentValues[kvp.Key] = kvp.Value; } } } foreach (var provider in module.InstanceArgumentProviders.Where(x => string.Equals(x.Instance, instance, StringComparison.InvariantCultureIgnoreCase))) { var values = provider.Arguments; if (values != null) { foreach (var kvp in values) { argumentValues[kvp.Key] = kvp.Value; } } } return(new EtlSessionArguments(argumentValues)); }
public static void UnloadModule(CommandContext commandContext, CompiledModule module) { commandContext.Logger.Debug("unloading module {Module}", module.Name); module.TaskTypes.Clear(); module.LoadContext?.Unload(); }
public static ExecutionResult LoadModule(CommandContext commandContext, string moduleName, bool forceCompilation, out CompiledModule module) { module = null; var moduleFolder = Path.Combine(commandContext.HostConfiguration.ModulesFolder, moduleName); if (!Directory.Exists(moduleFolder)) { commandContext.Logger.Write(LogEventLevel.Fatal, "can't find the module folder: {Folder}", moduleFolder); return(ExecutionResult.ModuleLoadError); } // read back actual folder name casing moduleFolder = Directory .GetDirectories(commandContext.HostConfiguration.ModulesFolder, moduleName, SearchOption.TopDirectoryOnly) .FirstOrDefault(); moduleName = Path.GetFileName(moduleFolder); var startedOn = Stopwatch.StartNew(); var useAppDomain = !forceCompilation && Debugger.IsAttached; if (useAppDomain) { commandContext.Logger.Information("loading module directly from AppDomain where namespace ends with '{Module}'", moduleName); var appDomainTasks = FindTypesFromAppDomain <IEtlTask>(moduleName); var startup = LoadInstancesFromAppDomain <IStartup>(moduleName).FirstOrDefault(); var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName); var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName); commandContext.Logger.Debug("finished in {Elapsed}", startedOn.Elapsed); module = new CompiledModule() { Name = moduleName, Folder = moduleFolder, Startup = startup, InstanceArgumentProviders = instanceConfigurationProviders, DefaultArgumentProviders = defaultConfigurationProviders, TaskTypes = appDomainTasks.Where(x => x.Name != null).ToList(), LoadContext = null, }; commandContext.Logger.Debug("{FlowCount} flows(s) found: {Task}", module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); commandContext.Logger.Debug("{TaskCount} task(s) found: {Task}", module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); return(ExecutionResult.Success); } commandContext.Logger.Information("compiling module from {Folder}", PathHelpers.GetFriendlyPathName(moduleFolder)); var selfFolder = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); var referenceAssemblyFolder = commandContext.HostConfiguration.CustomReferenceAssemblyFolder; if (string.IsNullOrEmpty(referenceAssemblyFolder)) { referenceAssemblyFolder = Directory.GetDirectories(@"c:\Program Files\dotnet\shared\Microsoft.NETCore.App\", "6.*") .OrderByDescending(x => new DirectoryInfo(x).CreationTime) .FirstOrDefault(); } commandContext.Logger.Information("using assemblies from {ReferenceAssemblyFolder}", referenceAssemblyFolder); var referenceDllFileNames = new List <string>(); referenceDllFileNames.AddRange(Directory.GetFiles(referenceAssemblyFolder, "System*.dll", SearchOption.TopDirectoryOnly)); referenceDllFileNames.AddRange(Directory.GetFiles(referenceAssemblyFolder, "netstandard.dll", SearchOption.TopDirectoryOnly)); var referenceFileNames = new List <string>(); referenceFileNames.AddRange(referenceDllFileNames.Where(x => !Path.GetFileNameWithoutExtension(x).EndsWith("Native", StringComparison.InvariantCultureIgnoreCase))); var localDllFileNames = Directory.GetFiles(selfFolder, "*.dll", SearchOption.TopDirectoryOnly) .Where(x => Path.GetFileName(x) != "FizzCode.EtLast.ConsoleHost.dll" && !Path.GetFileName(x).Equals("testhost.dll", StringComparison.InvariantCultureIgnoreCase)); referenceFileNames.AddRange(localDllFileNames); /*foreach (var fn in referenceFileNames) * Console.WriteLine(fn);*/ var metadataReferences = referenceFileNames .Distinct() .Select(fn => MetadataReference.CreateFromFile(fn)) .ToArray(); var csFileNames = Directory.GetFiles(moduleFolder, "*.cs", SearchOption.AllDirectories); var parseOptions = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.CSharp10); var syntaxTrees = csFileNames .Select(fn => SyntaxFactory.ParseSyntaxTree(SourceText.From(File.ReadAllText(fn)), parseOptions, fn)) .ToArray(); using (var assemblyStream = new MemoryStream()) { var id = Interlocked.Increment(ref _moduleAutoincrementId); var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, optimizationLevel: OptimizationLevel.Release, assemblyIdentityComparer: DesktopAssemblyIdentityComparer.Default); var compilation = CSharpCompilation.Create("compiled_" + id.ToString("D", CultureInfo.InvariantCulture) + ".dll", syntaxTrees, metadataReferences, compilationOptions); var result = compilation.Emit(assemblyStream); if (!result.Success) { var failures = result.Diagnostics.Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error); foreach (var error in failures) { commandContext.Logger.Write(LogEventLevel.Fatal, "syntax error in module: {ErrorMessage}", error.ToString()); } return(ExecutionResult.ModuleLoadError); } assemblyStream.Seek(0, SeekOrigin.Begin); var assemblyLoadContext = new AssemblyLoadContext(null, isCollectible: true); var assembly = assemblyLoadContext.LoadFromStream(assemblyStream); var compiledTasks = FindTypesFromAssembly <IEtlTask>(assembly); var compiledStartup = LoadInstancesFromAssembly <IStartup>(assembly).FirstOrDefault(); var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName); var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName); commandContext.Logger.Debug("compilation finished in {Elapsed}", startedOn.Elapsed); module = new CompiledModule() { Name = moduleName, Folder = moduleFolder, Startup = compiledStartup, InstanceArgumentProviders = instanceConfigurationProviders, DefaultArgumentProviders = defaultConfigurationProviders, TaskTypes = compiledTasks.Where(x => x.Name != null).ToList(), LoadContext = assemblyLoadContext, }; commandContext.Logger.Debug("{FlowCount} flows(s) found: {Task}", module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); commandContext.Logger.Debug("{TaskCount} task(s) found: {Task}", module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); return(ExecutionResult.Success); } }
public static ExecutionResult Execute(CommandContext commandContext, CompiledModule module, string[] commands) { var result = ExecutionResult.Success; var etlContext = new EtlContext(); try { var listeners = commandContext.HostConfiguration.GetSessionListeners(null); if (listeners?.Count > 0) { etlContext.Listeners.AddRange(listeners); } } catch (Exception ex) { var formattedMessage = ex.FormatExceptionWithDetails(); etlContext.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage); etlContext.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage); } var instance = Environment.MachineName; var arguments = GetArguments(module, instance); var environmentSettings = new EnvironmentSettings(); module.Startup.Configure(environmentSettings); var taskCreators = new Dictionary <string, Func <IEtlSessionArguments, IEtlTask> >(module.Startup.Commands, StringComparer.InvariantCultureIgnoreCase); var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture); var session = new EtlSession(sessionId, etlContext, arguments); etlContext.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout; var logger = CreateLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder); var opsLogger = CreateOpsLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder); var ioLogger = CreateIoLogger(environmentSettings, commandContext.DevLogFolder, commandContext.OpsLogFolder); var serilogAdapter = new EtlSessionSerilogAdapter(logger, opsLogger, ioLogger, commandContext.DevLogFolder, commandContext.OpsLogFolder); etlContext.Listeners.Add(serilogAdapter); serilogAdapter.Log(LogSeverity.Information, false, null, null, "session {SessionId} started", sessionId); if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url)) { etlContext.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url); } var sessionStartedOn = Stopwatch.StartNew(); var sessionExceptionCount = 0; var taskResults = new List <KeyValuePair <IEtlTask, TaskResult> >(); try { foreach (var command in commands) { IEtlTask task = null; if (taskCreators.TryGetValue(command, out var taskCreator)) { task = taskCreator.Invoke(arguments); } else { var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, command, StringComparison.InvariantCultureIgnoreCase)); if (taskType != null) { task = (IEtlTask)Activator.CreateInstance(taskType); } } if (task == null) { serilogAdapter.Log(LogSeverity.Error, false, null, null, "unknown task/flow type: " + command); break; } try { try { var taskResult = session.ExecuteTask(null, task); taskResults.Add(new KeyValuePair <IEtlTask, TaskResult>(task, taskResult)); sessionExceptionCount += taskResult.ExceptionCount; if (sessionExceptionCount > 0) { etlContext.Log(LogSeverity.Error, task, "failed, terminating execution"); result = ExecutionResult.ExecutionFailed; etlContext.Close(); break; // stop processing tasks } } catch (Exception ex) { etlContext.Log(LogSeverity.Error, task, "failed, terminating execution, reason: ", task.Statistics.RunTime, ex.Message); result = ExecutionResult.ExecutionFailed; etlContext.Close(); break; // stop processing tasks } } catch (TransactionAbortedException) { } LogTaskCounters(serilogAdapter, task); } session.Stop(); if (taskResults.Count > 0) { serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------"); serilogAdapter.Log(LogSeverity.Information, false, null, null, "SUMMARY"); serilogAdapter.Log(LogSeverity.Information, false, null, null, "-------"); var longestTaskName = taskResults.Max(x => x.Key.Name.Length); foreach (var kvp in taskResults) { LogTaskSummary(serilogAdapter, kvp.Key, kvp.Value, longestTaskName); } } etlContext.Close(); } catch (TransactionAbortedException) { } return(result); }