public static T Compile <T>(
ControlFlowGraph cfg,
OperandType[] funcArgTypes,
OperandType funcReturnType,
CompilerOptions options)
{
Logger.StartPass(PassName.Dominance);
Dominance.FindDominators(cfg);
Dominance.FindDominanceFrontiers(cfg);
Logger.EndPass(PassName.Dominance);
Logger.StartPass(PassName.SsaConstruction);
if ((options & CompilerOptions.SsaForm) != 0)
{
Ssa.Construct(cfg);
}
else
{
RegisterToLocal.Rename(cfg);
}
Logger.EndPass(PassName.SsaConstruction, cfg);
CompilerContext cctx = new CompilerContext(cfg, funcArgTypes, funcReturnType, options);
CompiledFunction func = CodeGenerator.Generate(cctx);
IntPtr codePtr = JitCache.Map(func);
return(Marshal.GetDelegateForFunctionPointer <T>(codePtr));
}
internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false)
{
if (!_funcs.TryGetValue(address, out TranslatedFunction func))
{
func = Translate(_memory, _jumpTable, address, mode, highCq: false);
TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
if (getFunc != func)
{
JitCache.Unmap(func.FuncPtr);
func = getFunc;
}
if (PtcProfiler.Enabled)
{
PtcProfiler.AddEntry(address, mode, highCq: false);
}
}
if (hintRejit && func.ShouldRejit())
{
_backgroundStack.Push(new RejitRequest(address, mode));
_backgroundTranslatorEvent.Set();
}
return(func);
}
public static T Compile <T>(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options)
{
CompiledFunction func = Compile(cfg, argTypes, retType, options);
IntPtr codePtr = JitCache.Map(func);
return(Marshal.GetDelegateForFunctionPointer <T>(codePtr));
}
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
_memory = memory;
_funcs = new ConcurrentDictionary <ulong, TranslatedFunction>();
_jumpTable = new JumpTable(allocator);
_backgroundQueue = new PriorityQueue <RejitRequest>(2);
_backgroundTranslatorEvent = new AutoResetEvent(false);
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
}
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
_allocator = allocator;
_memory = memory;
_funcs = new ConcurrentDictionary <ulong, TranslatedFunction>();
_oldFuncs = new ConcurrentQueue <KeyValuePair <ulong, IntPtr> >();
_backgroundStack = new ConcurrentStack <RejitRequest>();
_backgroundTranslatorEvent = new AutoResetEvent(false);
_backgroundTranslatorLock = new ReaderWriterLock();
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
}
private void ClearJitCache()
{
// Ensure no attempt will be made to compile new functions due to rejit.
ClearRejitQueue(allowRequeue: false);
foreach (var kv in _funcs)
{
JitCache.Unmap(kv.Value.FuncPtr);
}
_funcs.Clear();
while (_oldFuncs.TryDequeue(out var kv))
{
JitCache.Unmap(kv.Value);
}
}
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
_memory = memory;
_funcs = new ConcurrentDictionary <ulong, TranslatedFunction>();
_backgroundStack = new ConcurrentStack <RejitRequest>();
_backgroundTranslatorEvent = new AutoResetEvent(false);
_jumpTable = new JumpTable(allocator);
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
if (Ptc.State == PtcState.Enabled)
{
Ptc.LoadTranslations(_funcs, memory.PageTablePointer, _jumpTable);
}
}
private static unsafe RuntimeFunction *FunctionTableHandler(ulong controlPc, IntPtr context)
{
int offset = (int)((long)controlPc - context.ToInt64());
if (!JitCache.TryFind(offset, out JitCacheEntry funcEntry))
{
// Not found.
return(null);
}
var unwindInfo = funcEntry.UnwindInfo;
int codeIndex = 0;
int spOffset = unwindInfo.FixedAllocSize;
foreach (var entry in unwindInfo.PushEntries)
{
if (entry.Type == RegisterType.Vector)
{
spOffset -= 16;
}
}
for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
{
var entry = unwindInfo.PushEntries[index];
if (entry.Type == RegisterType.Vector)
{
ushort uwop = PackUwop(UnwindOperation.SaveXmm128, entry.StreamEndOffset, entry.Index);
_unwindInfo->UnwindCodes[codeIndex++] = uwop;
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)spOffset;
spOffset += 16;
}
}
_unwindInfo->UnwindCodes[0] = PackUwop(UnwindOperation.AllocLarge, unwindInfo.PrologueSize, 1);
_unwindInfo->UnwindCodes[1] = (ushort)(unwindInfo.FixedAllocSize >> 0);
_unwindInfo->UnwindCodes[2] = (ushort)(unwindInfo.FixedAllocSize >> 16);
codeIndex += 3;
for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
{
var entry = unwindInfo.PushEntries[index];
if (entry.Type == RegisterType.Integer)
{
ushort uwop = PackUwop(UnwindOperation.PushNonvol, entry.StreamEndOffset, entry.Index);
_unwindInfo->UnwindCodes[codeIndex++] = uwop;
}
}
_unwindInfo->VersionAndFlags = 1;
_unwindInfo->SizeOfProlog = (byte)unwindInfo.PrologueSize;
_unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
_unwindInfo->FrameRegister = 0;
_runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
_runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size);
_runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction;
return(_runtimeFunction);
}
private static unsafe RuntimeFunction *FunctionTableHandler(ulong controlPc, IntPtr context)
{
int offset = (int)((long)controlPc - context.ToInt64());
if (!JitCache.TryFind(offset, out JitCacheEntry funcEntry))
{
return(null); // Not found.
}
var unwindInfo = funcEntry.UnwindInfo;
int codeIndex = 0;
for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
{
var entry = unwindInfo.PushEntries[index];
switch (entry.PseudoOp)
{
case UnwindPseudoOp.SaveXmm128:
{
int stackOffset = entry.StackOffsetOrAllocSize;
Debug.Assert(stackOffset % 16 == 0);
if (stackOffset <= 0xFFFF0)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.SaveXmm128, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.SaveXmm128Far, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16);
}
break;
}
case UnwindPseudoOp.AllocStack:
{
int allocSize = entry.StackOffsetOrAllocSize;
Debug.Assert(allocSize % 8 == 0);
if (allocSize <= 128)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1);
}
else if (allocSize <= 0x7FFF8)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocLarge, entry.PrologOffset, 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocLarge, entry.PrologOffset, 1);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16);
}
break;
}
case UnwindPseudoOp.PushReg:
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.PushNonvol, entry.PrologOffset, entry.RegIndex);
break;
}
default: throw new NotImplementedException($"({nameof(entry.PseudoOp)} = {entry.PseudoOp})");
}
}
Debug.Assert(codeIndex <= MaxUnwindCodesArraySize);
_unwindInfo->VersionAndFlags = 1; // Flags: The function has no handler.
_unwindInfo->SizeOfProlog = (byte)unwindInfo.PrologSize;
_unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
_unwindInfo->FrameRegister = 0;
_runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
_runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size);
_runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction;
return(_runtimeFunction);
}
