/// <summary> /// Gets a type's encoded size in bytes. /// </summary> /// <param name="type">A type.</param> /// <param name="metadataSize">The object's metadata size.</param> /// <returns>The type's size in bytes, including the size of the metadata.</returns> public int SizeOfWithMetadata(IType type, out int metadataSize) { var ext = GCInterface.GetMetadataExtendedType(CompiledModule.ImportType(type), CompiledModule); metadataSize = (int)LLVM.OffsetOfElement(Target, ext, 1); return((int)LLVM.StoreSizeOfType(Target, ext)); }
/// <summary> /// Gets the offset of a particular field. /// </summary> /// <param name="field">A non-static field.</param> /// <returns>The field's offset in its parent type's layout.</returns> public int GetFieldOffset(IField field) { var parent = CompiledModule.ImportType(field.ParentType); var index = CompiledModule.GetFieldIndex(field); return((int)LLVM.OffsetOfElement(Target, parent, (uint)index)); }
public CompiledModule Compile() { if (compiled == null) { //special case for tests where we //don't have any AST if (ast != null) { Visit(ast); } byte[] init_bytes; byte[] code_bytes; Ip2SrcLine ip2src_line; Bake(out init_bytes, out code_bytes, out ip2src_line); compiled = new CompiledModule( module.name, module.symbols, constants, init_bytes, code_bytes, ip2src_line ); } return(compiled); }
public static void UnloadModule(Host host, CompiledModule module) { host.HostLogger.Debug("unloading module {Module}", module.Name); module.TaskTypes.Clear(); module.LoadContext?.Unload(); }
static void Dump(CompiledModule c) { if (c.initcode?.Length > 0) { Console.WriteLine("=== INIT ==="); Dump(c.initcode); } Console.WriteLine("=== CODE ==="); Dump(c.bytecode); }
public static int ConstIdx(CompiledModule cm, double num) { for (int i = 0; i < cm.constants.Count; ++i) { var cn = cm.constants[i]; if (cn.type == ConstType.FLT && cn.num == num) { return(i); } } throw new Exception("Constant not found: " + num); }
public static int ConstIdx(CompiledModule cm, string str) { for (int i = 0; i < cm.constants.Count; ++i) { var cn = cm.constants[i]; if (cn.type == ConstType.STR && cn.str == str) { return(i); } } throw new Exception("Constant not found: " + str); }
public static int ConstIdx(CompiledModule cm, bool v) { for (int i = 0; i < cm.constants.Count; ++i) { var cn = cm.constants[i]; if (cn.type == ConstType.BOOL && cn.num == (v ? 1 : 0)) { return(i); } } throw new Exception("Constant not found: " + v); }
public static int ConstIdx(CompiledModule cm, TypeProxy v) { for (int i = 0; i < cm.constants.Count; ++i) { var cn = cm.constants[i]; if (cn.type == ConstType.TPROXY && cn.tproxy.name == v.name) { return(i); } } throw new Exception("Constant not found: " + v); }
public static int ConstNullIdx(CompiledModule cm) { for (int i = 0; i < cm.constants.Count; ++i) { var cn = cm.constants[i]; if (cn.type == ConstType.NIL) { return(i); } } throw new Exception("Constant null not found"); }
public static void AssertEqual(CompiledModule ca, CompiledModule cb) { string cmp; if (!CompareCode(ca.initcode, cb.initcode, out cmp)) { Console.WriteLine(cmp); throw new Exception("Assertion failed: init bytes not equal"); } if (!CompareCode(ca.bytecode, cb.bytecode, out cmp)) { Console.WriteLine(cmp); throw new Exception("Assertion failed: bytes not equal"); } }
public CompiledModule Load(string module_name) { Entry ent; if (!name2entry.TryGetValue(module_name, out ent)) { return(null); } byte[] res = null; int res_len = 0; DecodeBin(ent, ref res, ref res_len); mod_stream.SetData(res, 0, res_len); return(CompiledModule.FromStream(types, mod_stream)); }
public static VM MakeVM(CompiledModule orig_cm, Types ts = null) { if (ts == null) { ts = new Types(); } //let's serialize/unserialize the compiled module so that //it's going to go thru the full compilation cycle var ms = new MemoryStream(); CompiledModule.ToStream(orig_cm, ms); var cm = CompiledModule.FromStream(ts, new MemoryStream(ms.GetBuffer()), add_symbols_to_types: true); var vm = new VM(ts); vm.RegisterModule(cm); return(vm); }
public static void DoWork(object data) { var sw = new Stopwatch(); sw.Start(); var w = (CompilerWorker)data; w.file2path.Clear(); w.file2compiled.Clear(); var imp = new Frontend.Importer(); imp.SetParsedCache(w.cache); imp.AddToIncludePath(w.inc_dir); int i = w.start; int cache_hit = 0; int cache_miss = 0; try { for (int cnt = 0; i < (w.start + w.count); ++i, ++cnt) { var file = w.files[i]; var compiled_file = GetCompiledCacheFile(w.cache_dir, file); var file_module = new Module(w.ts.globs, imp.FilePath2ModuleName(file), file); InterimResult interim; if (w.cache.file2interim.TryGetValue(file, out interim) && interim.use_file_cache) { ++cache_hit; w.file2path.Add(file, file_module.path); //TODO: load from the cached file? //w.file2symbols.Add(file, ...); } else { ++cache_miss; Frontend.Result front_res = null; if (interim.parsed != null) { front_res = Frontend.ProcessParsed(file_module, interim.parsed, w.ts, imp); } else { front_res = Frontend.ProcessFile(file, w.ts, imp); } front_res = w.postproc.Patch(front_res, file); w.file2path.Add(file, file_module.path); w.file2symbols.Add(file, front_res.module.symbols); var c = new Compiler(w.ts, front_res); var cm = c.Compile(); CompiledModule.ToFile(cm, compiled_file); } w.file2compiled.Add(file, compiled_file); } } catch (Exception e) { if (e is IError) { w.error = e; } else { //let's log unexpected exceptions immediately Console.Error.WriteLine(e.Message + " " + e.StackTrace); w.error = new BuildError(w.files[i], e); } } sw.Stop(); Console.WriteLine("BHL compiler {0} done(hit/miss:{2}/{3}, {1} sec)", w.id, Math.Round(sw.ElapsedMilliseconds / 1000.0f, 2), cache_hit, cache_miss); }
public static IExecutionResult Execute(Host host, CompiledModule module, string[] taskNames) { var executionResult = new ExecutionResult { TaskResults = new List <TaskExectionResult>(), }; var instance = Environment.MachineName; var arguments = new ArgumentCollection(module.DefaultArgumentProviders, module.InstanceArgumentProviders, instance); var environmentSettings = new EnvironmentSettings(); module.Startup.Configure(environmentSettings); var customTasks = new Dictionary <string, Func <IArgumentCollection, IEtlTask> >(module.Startup.CustomTasks, StringComparer.InvariantCultureIgnoreCase); var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture); var session = new EtlSession(sessionId, arguments); session.Context.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout; try { if (host.EtlContextListeners?.Count > 0) { foreach (var listenerCreator in host.EtlContextListeners) { session.Context.Listeners.Add(listenerCreator.Invoke(session)); } } } catch (Exception ex) { var formattedMessage = ex.FormatExceptionWithDetails(); session.Context.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage); session.Context.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage); } if (host.SerilogForModulesEnabled) { if (environmentSettings.FileLogSettings.Enabled || environmentSettings.ConsoleLogSettings.Enabled || !string.IsNullOrEmpty(environmentSettings.SeqSettings.Url)) { var serilogAdapter = new EtlSessionSerilogAdapter(environmentSettings, host.DevLogFolder, host.OpsLogFolder); session.Context.Listeners.Add(serilogAdapter); } } session.Context.Log(LogSeverity.Information, null, "session {SessionId} started", sessionId); if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url)) { session.Context.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url); } var sessionStartedOn = Stopwatch.StartNew(); var sessionExceptions = new List <Exception>(); var taskResults = new List <TaskExectionResult>(); try { foreach (var taskName in taskNames) { IEtlTask task = null; if (customTasks.TryGetValue(taskName, out var taskCreator)) { task = taskCreator.Invoke(arguments); } else { var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, taskName, StringComparison.InvariantCultureIgnoreCase)); if (taskType != null) { task = (IEtlTask)Activator.CreateInstance(taskType); } } if (task == null) { session.Context.Log(LogSeverity.Error, null, "unknown task/flow type: " + taskName); break; } try { try { var taskResult = session.ExecuteTask(null, task) as ProcessResult; taskResults.Add(new TaskExectionResult(task, taskResult)); executionResult.TaskResults.Add(new TaskExectionResult(task, taskResult)); sessionExceptions.AddRange(taskResult.Exceptions); if (sessionExceptions.Count > 0) { session.Context.Log(LogSeverity.Error, task, "failed, terminating execution"); executionResult.Status = ExecutionStatusCode.ExecutionFailed; session.Context.Close(); break; // stop processing tasks } } catch (Exception ex) { session.Context.Log(LogSeverity.Error, task, "failed, terminating execution, reason: {0}", ex.Message); executionResult.Status = ExecutionStatusCode.ExecutionFailed; session.Context.Close(); break; // stop processing tasks } } catch (TransactionAbortedException) { } LogTaskCounters(session.Context, task); } session.Stop(); if (taskResults.Count > 0) { session.Context.Log(LogSeverity.Information, null, "-------"); session.Context.Log(LogSeverity.Information, null, "SUMMARY"); session.Context.Log(LogSeverity.Information, null, "-------"); var longestTaskName = taskResults.Max(x => x.TaskName.Length); foreach (var taskResult in taskResults) { LogTaskSummary(session.Context, taskResult, longestTaskName); } } session.Context.Close(); } catch (TransactionAbortedException) { } return(executionResult); }
public static ExecutionStatusCode LoadModule(Host host, string moduleName, bool forceCompilation, out CompiledModule module) { module = null; var moduleFolder = Path.Combine(host.ModulesFolder, moduleName); if (!Directory.Exists(moduleFolder)) { host.HostLogger.Write(LogEventLevel.Fatal, "can't find the module folder: {Folder}", moduleFolder); return(ExecutionStatusCode.ModuleLoadError); } // read back actual folder name casing moduleFolder = Directory .GetDirectories(host.ModulesFolder, moduleName, SearchOption.TopDirectoryOnly) .FirstOrDefault(); moduleName = Path.GetFileName(moduleFolder); var startedOn = Stopwatch.StartNew(); var useAppDomain = !forceCompilation && Debugger.IsAttached; if (useAppDomain) { host.HostLogger.Information("loading module directly from AppDomain where namespace ends with '{Module}'", moduleName); ForceLoadLocalDllsToAppDomain(); var appDomainTasks = FindTypesFromAppDomain <IEtlTask>(moduleName); var startup = LoadInstancesFromAppDomain <IStartup>(moduleName).FirstOrDefault(); var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName); var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName); host.HostLogger.Debug("finished in {Elapsed}", startedOn.Elapsed); module = new CompiledModule() { Name = moduleName, Folder = moduleFolder, Startup = startup, InstanceArgumentProviders = instanceConfigurationProviders, DefaultArgumentProviders = defaultConfigurationProviders, TaskTypes = appDomainTasks.Where(x => x.Name != null).ToList(), LoadContext = null, }; host.HostLogger.Debug("{FlowCount} flows(s) found: {Task}", module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); host.HostLogger.Debug("{TaskCount} task(s) found: {Task}", module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); return(ExecutionStatusCode.Success); } host.HostLogger.Information("compiling module from {Folder}", PathHelpers.GetFriendlyPathName(moduleFolder)); var metadataReferences = host.GetReferenceAssemblyFileNames() .Select(fn => MetadataReference.CreateFromFile(fn)) .ToArray(); var csFileNames = Directory.GetFiles(moduleFolder, "*.cs", SearchOption.AllDirectories).ToList(); var globalCsFileName = Path.Combine(host.ModulesFolder, "Global.cs"); if (File.Exists(globalCsFileName)) { csFileNames.Add(globalCsFileName); } var parseOptions = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.CSharp10); var syntaxTrees = csFileNames .Select(fn => SyntaxFactory.ParseSyntaxTree(SourceText.From(File.ReadAllText(fn)), parseOptions, fn)) .ToArray(); using (var assemblyStream = new MemoryStream()) { var id = Interlocked.Increment(ref _moduleAutoincrementId); var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, optimizationLevel: OptimizationLevel.Release, assemblyIdentityComparer: DesktopAssemblyIdentityComparer.Default); var compilation = CSharpCompilation.Create("compiled_" + id.ToString("D", CultureInfo.InvariantCulture) + ".dll", syntaxTrees, metadataReferences, compilationOptions); var result = compilation.Emit(assemblyStream); if (!result.Success) { var failures = result.Diagnostics.Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error); foreach (var error in failures) { host.HostLogger.Write(LogEventLevel.Fatal, "syntax error in module: {ErrorMessage}", error.ToString()); } return(ExecutionStatusCode.ModuleLoadError); } assemblyStream.Seek(0, SeekOrigin.Begin); var assemblyLoadContext = new AssemblyLoadContext(null, isCollectible: true); var assembly = assemblyLoadContext.LoadFromStream(assemblyStream); var compiledTasks = FindTypesFromAssembly <IEtlTask>(assembly); var compiledStartup = LoadInstancesFromAssembly <IStartup>(assembly).FirstOrDefault(); var instanceConfigurationProviders = LoadInstancesFromAssembly <IInstanceArgumentProvider>(assembly); var defaultConfigurationProviders = LoadInstancesFromAssembly <IDefaultArgumentProvider>(assembly); host.HostLogger.Debug("compilation finished in {Elapsed}", startedOn.Elapsed); module = new CompiledModule() { Name = moduleName, Folder = moduleFolder, Startup = compiledStartup, InstanceArgumentProviders = instanceConfigurationProviders, DefaultArgumentProviders = defaultConfigurationProviders, TaskTypes = compiledTasks.Where(x => x.Name != null).ToList(), LoadContext = assemblyLoadContext, }; host.HostLogger.Debug("{FlowCount} flows(s) found: {Task}", module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); host.HostLogger.Debug("{TaskCount} task(s) found: {Task}", module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray()); return(ExecutionStatusCode.Success); } }
/// <summary> /// Gets a type's encoded size in bytes. /// </summary> /// <param name="type">A type.</param> /// <returns>The type's size in bytes.</returns> public int SizeOf(IType type) { return((int)LLVM.StoreSizeOfType(Target, CompiledModule.ImportType(type))); }
public static void AssertEqual(CompiledModule ca, Compiler cb) { AssertEqual(ca, cb.Compile()); }