/// <summary>
/// Gets a type's encoded size in bytes.
/// </summary>
/// <param name="type">A type.</param>
/// <param name="metadataSize">The object's metadata size.</param>
/// <returns>The type's size in bytes, including the size of the metadata.</returns>
public int SizeOfWithMetadata(IType type, out int metadataSize)
{
var ext = GCInterface.GetMetadataExtendedType(CompiledModule.ImportType(type), CompiledModule);
metadataSize = (int)LLVM.OffsetOfElement(Target, ext, 1);
return((int)LLVM.StoreSizeOfType(Target, ext));
}
/// <summary>
/// Gets the offset of a particular field.
/// </summary>
/// <param name="field">A non-static field.</param>
/// <returns>The field's offset in its parent type's layout.</returns>
public int GetFieldOffset(IField field)
{
var parent = CompiledModule.ImportType(field.ParentType);
var index = CompiledModule.GetFieldIndex(field);
return((int)LLVM.OffsetOfElement(Target, parent, (uint)index));
}
public CompiledModule Compile()
{
if (compiled == null)
{
//special case for tests where we
//don't have any AST
if (ast != null)
{
Visit(ast);
}
byte[] init_bytes;
byte[] code_bytes;
Ip2SrcLine ip2src_line;
Bake(out init_bytes, out code_bytes, out ip2src_line);
compiled = new CompiledModule(
module.name,
module.symbols,
constants,
init_bytes,
code_bytes,
ip2src_line
);
}
return(compiled);
}
public static void UnloadModule(Host host, CompiledModule module)
{
host.HostLogger.Debug("unloading module {Module}", module.Name);
module.TaskTypes.Clear();
module.LoadContext?.Unload();
}
static void Dump(CompiledModule c)
{
if (c.initcode?.Length > 0)
{
Console.WriteLine("=== INIT ===");
Dump(c.initcode);
}
Console.WriteLine("=== CODE ===");
Dump(c.bytecode);
}
public static int ConstIdx(CompiledModule cm, double num)
{
for (int i = 0; i < cm.constants.Count; ++i)
{
var cn = cm.constants[i];
if (cn.type == ConstType.FLT && cn.num == num)
{
return(i);
}
}
throw new Exception("Constant not found: " + num);
}
public static int ConstIdx(CompiledModule cm, string str)
{
for (int i = 0; i < cm.constants.Count; ++i)
{
var cn = cm.constants[i];
if (cn.type == ConstType.STR && cn.str == str)
{
return(i);
}
}
throw new Exception("Constant not found: " + str);
}
public static int ConstIdx(CompiledModule cm, bool v)
{
for (int i = 0; i < cm.constants.Count; ++i)
{
var cn = cm.constants[i];
if (cn.type == ConstType.BOOL && cn.num == (v ? 1 : 0))
{
return(i);
}
}
throw new Exception("Constant not found: " + v);
}
public static int ConstIdx(CompiledModule cm, TypeProxy v)
{
for (int i = 0; i < cm.constants.Count; ++i)
{
var cn = cm.constants[i];
if (cn.type == ConstType.TPROXY && cn.tproxy.name == v.name)
{
return(i);
}
}
throw new Exception("Constant not found: " + v);
}
public static int ConstNullIdx(CompiledModule cm)
{
for (int i = 0; i < cm.constants.Count; ++i)
{
var cn = cm.constants[i];
if (cn.type == ConstType.NIL)
{
return(i);
}
}
throw new Exception("Constant null not found");
}
public static void AssertEqual(CompiledModule ca, CompiledModule cb)
{
string cmp;
if (!CompareCode(ca.initcode, cb.initcode, out cmp))
{
Console.WriteLine(cmp);
throw new Exception("Assertion failed: init bytes not equal");
}
if (!CompareCode(ca.bytecode, cb.bytecode, out cmp))
{
Console.WriteLine(cmp);
throw new Exception("Assertion failed: bytes not equal");
}
}
public CompiledModule Load(string module_name)
{
Entry ent;
if (!name2entry.TryGetValue(module_name, out ent))
{
return(null);
}
byte[] res = null;
int res_len = 0;
DecodeBin(ent, ref res, ref res_len);
mod_stream.SetData(res, 0, res_len);
return(CompiledModule.FromStream(types, mod_stream));
}
public static VM MakeVM(CompiledModule orig_cm, Types ts = null)
{
if (ts == null)
{
ts = new Types();
}
//let's serialize/unserialize the compiled module so that
//it's going to go thru the full compilation cycle
var ms = new MemoryStream();
CompiledModule.ToStream(orig_cm, ms);
var cm = CompiledModule.FromStream(ts, new MemoryStream(ms.GetBuffer()), add_symbols_to_types: true);
var vm = new VM(ts);
vm.RegisterModule(cm);
return(vm);
}
public static void DoWork(object data)
{
var sw = new Stopwatch();
sw.Start();
var w = (CompilerWorker)data;
w.file2path.Clear();
w.file2compiled.Clear();
var imp = new Frontend.Importer();
imp.SetParsedCache(w.cache);
imp.AddToIncludePath(w.inc_dir);
int i = w.start;
int cache_hit = 0;
int cache_miss = 0;
try
{
for (int cnt = 0; i < (w.start + w.count); ++i, ++cnt)
{
var file = w.files[i];
var compiled_file = GetCompiledCacheFile(w.cache_dir, file);
var file_module = new Module(w.ts.globs, imp.FilePath2ModuleName(file), file);
InterimResult interim;
if (w.cache.file2interim.TryGetValue(file, out interim) &&
interim.use_file_cache)
{
++cache_hit;
w.file2path.Add(file, file_module.path);
//TODO: load from the cached file?
//w.file2symbols.Add(file, ...);
}
else
{
++cache_miss;
Frontend.Result front_res = null;
if (interim.parsed != null)
{
front_res = Frontend.ProcessParsed(file_module, interim.parsed, w.ts, imp);
}
else
{
front_res = Frontend.ProcessFile(file, w.ts, imp);
}
front_res = w.postproc.Patch(front_res, file);
w.file2path.Add(file, file_module.path);
w.file2symbols.Add(file, front_res.module.symbols);
var c = new Compiler(w.ts, front_res);
var cm = c.Compile();
CompiledModule.ToFile(cm, compiled_file);
}
w.file2compiled.Add(file, compiled_file);
}
}
catch (Exception e)
{
if (e is IError)
{
w.error = e;
}
else
{
//let's log unexpected exceptions immediately
Console.Error.WriteLine(e.Message + " " + e.StackTrace);
w.error = new BuildError(w.files[i], e);
}
}
sw.Stop();
Console.WriteLine("BHL compiler {0} done(hit/miss:{2}/{3}, {1} sec)", w.id, Math.Round(sw.ElapsedMilliseconds / 1000.0f, 2), cache_hit, cache_miss);
}
public static IExecutionResult Execute(Host host, CompiledModule module, string[] taskNames)
{
var executionResult = new ExecutionResult
{
TaskResults = new List <TaskExectionResult>(),
};
var instance = Environment.MachineName;
var arguments = new ArgumentCollection(module.DefaultArgumentProviders, module.InstanceArgumentProviders, instance);
var environmentSettings = new EnvironmentSettings();
module.Startup.Configure(environmentSettings);
var customTasks = new Dictionary <string, Func <IArgumentCollection, IEtlTask> >(module.Startup.CustomTasks, StringComparer.InvariantCultureIgnoreCase);
var sessionId = "s" + DateTime.Now.ToString("yyMMdd-HHmmss-ff", CultureInfo.InvariantCulture);
var session = new EtlSession(sessionId, arguments);
session.Context.TransactionScopeTimeout = environmentSettings.TransactionScopeTimeout;
try
{
if (host.EtlContextListeners?.Count > 0)
{
foreach (var listenerCreator in host.EtlContextListeners)
{
session.Context.Listeners.Add(listenerCreator.Invoke(session));
}
}
}
catch (Exception ex)
{
var formattedMessage = ex.FormatExceptionWithDetails();
session.Context.Log(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
session.Context.LogOps(LogSeverity.Fatal, null, "{ErrorMessage}", formattedMessage);
}
if (host.SerilogForModulesEnabled)
{
if (environmentSettings.FileLogSettings.Enabled || environmentSettings.ConsoleLogSettings.Enabled || !string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
var serilogAdapter = new EtlSessionSerilogAdapter(environmentSettings, host.DevLogFolder, host.OpsLogFolder);
session.Context.Listeners.Add(serilogAdapter);
}
}
session.Context.Log(LogSeverity.Information, null, "session {SessionId} started", sessionId);
if (!string.IsNullOrEmpty(environmentSettings.SeqSettings.Url))
{
session.Context.Log(LogSeverity.Debug, null, "all session logs will be sent to SEQ listening on {SeqUrl}", environmentSettings.SeqSettings.Url);
}
var sessionStartedOn = Stopwatch.StartNew();
var sessionExceptions = new List <Exception>();
var taskResults = new List <TaskExectionResult>();
try
{
foreach (var taskName in taskNames)
{
IEtlTask task = null;
if (customTasks.TryGetValue(taskName, out var taskCreator))
{
task = taskCreator.Invoke(arguments);
}
else
{
var taskType = module.TaskTypes.Find(x => string.Equals(x.Name, taskName, StringComparison.InvariantCultureIgnoreCase));
if (taskType != null)
{
task = (IEtlTask)Activator.CreateInstance(taskType);
}
}
if (task == null)
{
session.Context.Log(LogSeverity.Error, null, "unknown task/flow type: " + taskName);
break;
}
try
{
try
{
var taskResult = session.ExecuteTask(null, task) as ProcessResult;
taskResults.Add(new TaskExectionResult(task, taskResult));
executionResult.TaskResults.Add(new TaskExectionResult(task, taskResult));
sessionExceptions.AddRange(taskResult.Exceptions);
if (sessionExceptions.Count > 0)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution");
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (Exception ex)
{
session.Context.Log(LogSeverity.Error, task, "failed, terminating execution, reason: {0}", ex.Message);
executionResult.Status = ExecutionStatusCode.ExecutionFailed;
session.Context.Close();
break; // stop processing tasks
}
}
catch (TransactionAbortedException)
{
}
LogTaskCounters(session.Context, task);
}
session.Stop();
if (taskResults.Count > 0)
{
session.Context.Log(LogSeverity.Information, null, "-------");
session.Context.Log(LogSeverity.Information, null, "SUMMARY");
session.Context.Log(LogSeverity.Information, null, "-------");
var longestTaskName = taskResults.Max(x => x.TaskName.Length);
foreach (var taskResult in taskResults)
{
LogTaskSummary(session.Context, taskResult, longestTaskName);
}
}
session.Context.Close();
}
catch (TransactionAbortedException)
{
}
return(executionResult);
}
public static ExecutionStatusCode LoadModule(Host host, string moduleName, bool forceCompilation, out CompiledModule module)
{
module = null;
var moduleFolder = Path.Combine(host.ModulesFolder, moduleName);
if (!Directory.Exists(moduleFolder))
{
host.HostLogger.Write(LogEventLevel.Fatal, "can't find the module folder: {Folder}", moduleFolder);
return(ExecutionStatusCode.ModuleLoadError);
}
// read back actual folder name casing
moduleFolder = Directory
.GetDirectories(host.ModulesFolder, moduleName, SearchOption.TopDirectoryOnly)
.FirstOrDefault();
moduleName = Path.GetFileName(moduleFolder);
var startedOn = Stopwatch.StartNew();
var useAppDomain = !forceCompilation && Debugger.IsAttached;
if (useAppDomain)
{
host.HostLogger.Information("loading module directly from AppDomain where namespace ends with '{Module}'", moduleName);
ForceLoadLocalDllsToAppDomain();
var appDomainTasks = FindTypesFromAppDomain <IEtlTask>(moduleName);
var startup = LoadInstancesFromAppDomain <IStartup>(moduleName).FirstOrDefault();
var instanceConfigurationProviders = LoadInstancesFromAppDomain <IInstanceArgumentProvider>(moduleName);
var defaultConfigurationProviders = LoadInstancesFromAppDomain <IDefaultArgumentProvider>(moduleName);
host.HostLogger.Debug("finished in {Elapsed}", startedOn.Elapsed);
module = new CompiledModule()
{
Name = moduleName,
Folder = moduleFolder,
Startup = startup,
InstanceArgumentProviders = instanceConfigurationProviders,
DefaultArgumentProviders = defaultConfigurationProviders,
TaskTypes = appDomainTasks.Where(x => x.Name != null).ToList(),
LoadContext = null,
};
host.HostLogger.Debug("{FlowCount} flows(s) found: {Task}",
module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
host.HostLogger.Debug("{TaskCount} task(s) found: {Task}",
module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
return(ExecutionStatusCode.Success);
}
host.HostLogger.Information("compiling module from {Folder}", PathHelpers.GetFriendlyPathName(moduleFolder));
var metadataReferences = host.GetReferenceAssemblyFileNames()
.Select(fn => MetadataReference.CreateFromFile(fn))
.ToArray();
var csFileNames = Directory.GetFiles(moduleFolder, "*.cs", SearchOption.AllDirectories).ToList();
var globalCsFileName = Path.Combine(host.ModulesFolder, "Global.cs");
if (File.Exists(globalCsFileName))
{
csFileNames.Add(globalCsFileName);
}
var parseOptions = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.CSharp10);
var syntaxTrees = csFileNames
.Select(fn => SyntaxFactory.ParseSyntaxTree(SourceText.From(File.ReadAllText(fn)), parseOptions, fn))
.ToArray();
using (var assemblyStream = new MemoryStream())
{
var id = Interlocked.Increment(ref _moduleAutoincrementId);
var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary,
optimizationLevel: OptimizationLevel.Release,
assemblyIdentityComparer: DesktopAssemblyIdentityComparer.Default);
var compilation = CSharpCompilation.Create("compiled_" + id.ToString("D", CultureInfo.InvariantCulture) + ".dll", syntaxTrees, metadataReferences, compilationOptions);
var result = compilation.Emit(assemblyStream);
if (!result.Success)
{
var failures = result.Diagnostics.Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error);
foreach (var error in failures)
{
host.HostLogger.Write(LogEventLevel.Fatal, "syntax error in module: {ErrorMessage}", error.ToString());
}
return(ExecutionStatusCode.ModuleLoadError);
}
assemblyStream.Seek(0, SeekOrigin.Begin);
var assemblyLoadContext = new AssemblyLoadContext(null, isCollectible: true);
var assembly = assemblyLoadContext.LoadFromStream(assemblyStream);
var compiledTasks = FindTypesFromAssembly <IEtlTask>(assembly);
var compiledStartup = LoadInstancesFromAssembly <IStartup>(assembly).FirstOrDefault();
var instanceConfigurationProviders = LoadInstancesFromAssembly <IInstanceArgumentProvider>(assembly);
var defaultConfigurationProviders = LoadInstancesFromAssembly <IDefaultArgumentProvider>(assembly);
host.HostLogger.Debug("compilation finished in {Elapsed}", startedOn.Elapsed);
module = new CompiledModule()
{
Name = moduleName,
Folder = moduleFolder,
Startup = compiledStartup,
InstanceArgumentProviders = instanceConfigurationProviders,
DefaultArgumentProviders = defaultConfigurationProviders,
TaskTypes = compiledTasks.Where(x => x.Name != null).ToList(),
LoadContext = assemblyLoadContext,
};
host.HostLogger.Debug("{FlowCount} flows(s) found: {Task}",
module.TaskTypes.Count(x => x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
host.HostLogger.Debug("{TaskCount} task(s) found: {Task}",
module.TaskTypes.Count(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))), module.TaskTypes.Where(x => !x.IsAssignableTo(typeof(AbstractEtlFlow))).Select(task => task.Name).ToArray());
return(ExecutionStatusCode.Success);
}
}
/// <summary>
/// Gets a type's encoded size in bytes.
/// </summary>
/// <param name="type">A type.</param>
/// <returns>The type's size in bytes.</returns>
public int SizeOf(IType type)
{
return((int)LLVM.StoreSizeOfType(Target, CompiledModule.ImportType(type)));
}
public static void AssertEqual(CompiledModule ca, Compiler cb)
{
AssertEqual(ca, cb.Compile());
}
