private static void EmitVirtualCallOrJump(ILEmitterCtx context, bool isJump)
{
if (context.Tier == TranslationTier.Tier0)
{
context.Emit(OpCodes.Dup);
context.EmitSttmp();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitFieldLoad(typeof(CpuThreadState).GetField(nameof(CpuThreadState.CurrentTranslator),
BindingFlags.Instance |
BindingFlags.NonPublic));
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(Translator), nameof(Translator.TranslateVirtualSubroutine));
context.Emit(OpCodes.Ret);
}
else
{
context.EmitSttmp();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitFieldLoad(typeof(CpuThreadState).GetField(nameof(CpuThreadState.CurrentTranslator),
BindingFlags.Instance |
BindingFlags.NonPublic));
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(Translator), nameof(Translator.GetOrTranslateVirtualSubroutine));
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
if (isJump)
{
//The tail prefix allows the JIT to jump to the next function,
//while releasing the stack space used by the current one.
//This is ideal for BR ARM instructions, which are
//basically indirect tail calls.
context.Emit(OpCodes.Tailcall);
}
MethodInfo mthdInfo = typeof(ArmSubroutine).GetMethod("Invoke");
context.EmitCall(mthdInfo, isVirtual: true);
if (!isJump)
{
EmitContinueOrReturnCheck(context);
}
else
{
context.Emit(OpCodes.Ret);
}
}
}
private static void EmitRrxC(ILEmitterCtx context, bool setCarry)
{
// Rotate right by 1 with carry.
if (setCarry)
{
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(1);
context.Emit(OpCodes.And);
context.EmitSttmp();
}
context.EmitLsr(1);
context.EmitLdflg((int)PState.CBit);
context.EmitLsl(31);
context.Emit(OpCodes.Or);
if (setCarry)
{
context.EmitLdtmp();
context.EmitStflg((int)PState.CBit);
}
}
public static void EmitVectorWidenOpByElem(ILEmitterCtx context, Action emit, int elem, bool ternary, bool signed)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
int elems = 8 >> op.Size;
int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0;
EmitVectorExtract(context, op.Rm, elem, op.Size, signed);
context.EmitSttmp();
for (int index = 0; index < elems; index++)
{
if (ternary)
{
EmitVectorExtract(context, op.Rd, index, op.Size + 1, signed);
}
EmitVectorExtract(context, op.Rn, part + index, op.Size, signed);
context.EmitLdtmp();
emit();
EmitVectorInsertTmp(context, index, op.Size + 1);
}
context.EmitLdvectmp();
context.EmitStvec(op.Rd);
}
public static void EmitVectorOpByElem(ILEmitterCtx context, Action emit, int elem, bool ternary, bool signed)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
int bytes = op.GetBitsCount() >> 3;
int elems = bytes >> op.Size;
EmitVectorExtract(context, op.Rm, elem, op.Size, signed);
context.EmitSttmp();
for (int index = 0; index < elems; index++)
{
if (ternary)
{
EmitVectorExtract(context, op.Rd, index, op.Size, signed);
}
EmitVectorExtract(context, op.Rn, index, op.Size, signed);
context.EmitLdtmp();
emit();
EmitVectorInsert(context, op.Rd, index, op.Size);
}
if (op.RegisterSize == RegisterSize.Simd64)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
private static void EmitLoadAddress(ILEmitterCtx context)
{
switch (context.CurrOp)
{
case OpCodeMemImm64 op:
context.EmitLdint(op.Rn);
if (!op.PostIdx)
{
// Pre-indexing.
context.EmitLdc_I(op.Imm);
context.Emit(OpCodes.Add);
}
break;
case OpCodeMemReg64 op:
context.EmitLdint(op.Rn);
context.EmitLdintzr(op.Rm);
context.EmitCast(op.IntType);
if (op.Shift)
{
context.EmitLsl(op.Size);
}
context.Emit(OpCodes.Add);
break;
}
// Save address to Scratch var since the register value may change.
context.Emit(OpCodes.Dup);
context.EmitSttmp();
}
public static void Ldm(ILEmitterCtx context)
{
OpCode32MemMult op = (OpCode32MemMult)context.CurrOp;
EmitLoadFromRegister(context, op.Rn);
bool writesToPc = (op.RegisterMask & (1 << RegisterAlias.Aarch32Pc)) != 0;
bool writeBack = op.PostOffset != 0 && (op.Rn != RegisterAlias.Aarch32Pc || !writesToPc);
if (writeBack)
{
context.Emit(OpCodes.Dup);
}
context.EmitLdc_I4(op.Offset);
context.Emit(OpCodes.Add);
context.EmitSttmp();
if (writeBack)
{
context.EmitLdc_I4(op.PostOffset);
context.Emit(OpCodes.Add);
EmitStoreToRegister(context, op.Rn);
}
int mask = op.RegisterMask;
int offset = 0;
for (int register = 0; mask != 0; mask >>= 1, register++)
{
if ((mask & 1) != 0)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I4(offset);
context.Emit(OpCodes.Add);
EmitReadZxCall(context, WordSizeLog2);
EmitStoreToRegister(context, register);
offset += 4;
}
}
}
public static void Stm(ILEmitterCtx context)
{
OpCode32MemMult op = (OpCode32MemMult)context.CurrOp;
EmitLoadFromRegister(context, op.Rn);
context.EmitLdc_I4(op.Offset);
context.Emit(OpCodes.Add);
context.EmitSttmp();
int mask = op.RegisterMask;
int offset = 0;
for (int register = 0; mask != 0; mask >>= 1, register++)
{
if ((mask & 1) != 0)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I4(offset);
context.Emit(OpCodes.Add);
EmitLoadFromRegister(context, register);
EmitWriteCall(context, WordSizeLog2);
//Note: If Rn is also specified on the register list,
//and Rn is the first register on this list, then the
//value that is written to memory is the unmodified value,
//before the write back. If it is on the list, but it's
//not the first one, then the value written to memory
//varies between CPUs.
if (offset == 0 && op.PostOffset != 0)
{
//Emit write back after the first write.
EmitLoadFromRegister(context, op.Rn);
context.EmitLdc_I4(op.PostOffset);
context.Emit(OpCodes.Add);
EmitStoreToRegister(context, op.Rn);
}
offset += 4;
}
}
}
public static void Blr(ILEmitterCtx context)
{
OpCodeBReg64 op = (OpCodeBReg64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitSttmp();
context.EmitLdc_I(op.Position + 4);
context.EmitStint(CpuThreadState.LrIndex);
context.EmitStoreState();
context.EmitLdtmp();
context.Emit(OpCodes.Ret);
}
public static void Bsl_V(ILEmitterCtx context)
{
if (Optimizations.UseSse2)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
Type[] types = new Type[]
{
VectorUIntTypesPerSizeLog2[op.Size],
VectorUIntTypesPerSizeLog2[op.Size]
};
EmitLdvecWithUnsignedCast(context, op.Rn, op.Size);
EmitLdvecWithUnsignedCast(context, op.Rm, op.Size);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.Xor), types));
EmitLdvecWithUnsignedCast(context, op.Rd, op.Size);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.And), types));
EmitLdvecWithUnsignedCast(context, op.Rm, op.Size);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.Xor), types));
EmitStvecWithUnsignedCast(context, op.Rd, op.Size);
if (op.RegisterSize == RegisterSize.Simd64)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
else
{
EmitVectorTernaryOpZx(context, () =>
{
context.EmitSttmp();
context.EmitLdtmp();
context.Emit(OpCodes.Xor);
context.Emit(OpCodes.And);
context.EmitLdtmp();
context.Emit(OpCodes.Xor);
});
}
}
public static void Bsl_V(ILEmitterCtx context)
{
if (Optimizations.UseSse2)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
Type[] typesXorAnd = new Type[] { typeof(Vector128 <byte>), typeof(Vector128 <byte>) };
context.EmitLdvec(op.Rm);
context.Emit(OpCodes.Dup);
context.EmitLdvec(op.Rn);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.Xor), typesXorAnd));
context.EmitLdvec(op.Rd);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.And), typesXorAnd));
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.Xor), typesXorAnd));
context.EmitStvec(op.Rd);
if (op.RegisterSize == RegisterSize.Simd64)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
else
{
EmitVectorTernaryOpZx(context, () =>
{
context.EmitSttmp();
context.EmitLdtmp();
context.Emit(OpCodes.Xor);
context.Emit(OpCodes.And);
context.EmitLdtmp();
context.Emit(OpCodes.Xor);
});
}
}
private static void EmitLoad(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
if (exclusive)
{
EmitMemoryCall(context, nameof(MemoryManager.SetExclusive), op.Rn);
}
context.EmitLdint(op.Rn);
context.EmitSttmp();
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, op.Size);
context.EmitStintzr(op.Rt);
if (pair)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
EmitReadZxCall(context, op.Size);
context.EmitStintzr(op.Rt2);
}
}
public static void EmitAdcsCCheck(ILEmitterCtx context)
{
// C = (Rd == Rn && CIn) || Rd < Rn
context.EmitSttmp();
context.EmitLdtmp();
context.EmitLdtmp();
EmitAluLoadRn(context);
context.Emit(OpCodes.Ceq);
context.EmitLdflg((int)PState.CBit);
context.Emit(OpCodes.And);
context.EmitLdtmp();
EmitAluLoadRn(context);
context.Emit(OpCodes.Clt_Un);
context.Emit(OpCodes.Or);
context.EmitStflg((int)PState.CBit);
}
private static void EmitLoadOrStore(ILEmitterCtx context, int size, AccessType accType)
{
OpCode32Mem op = (OpCode32Mem)context.CurrOp;
if (op.Index || op.WBack)
{
EmitLoadFromRegister(context, op.Rn);
context.EmitLdc_I4(op.Imm);
context.Emit(op.Add ? OpCodes.Add : OpCodes.Sub);
context.EmitSttmp();
}
if (op.Index)
{
context.EmitLdtmp();
}
else
{
EmitLoadFromRegister(context, op.Rn);
}
if ((accType & AccessType.Load) != 0)
{
if ((accType & AccessType.Signed) != 0)
{
EmitReadSx32Call(context, size);
}
else
{
EmitReadZxCall(context, size);
}
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Dup);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.MarkLabel(lblEnd);
}
else
{
EmitStoreToRegister(context, op.Rt);
}
}
else
{
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
EmitLoadFromRegister(context, op.Rt);
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Conv_U8);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.EmitLsl(32);
context.Emit(OpCodes.Or);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsl(32);
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.Emit(OpCodes.Or);
context.MarkLabel(lblEnd);
}
EmitWriteCall(context, size);
}
}
private static void EmitLoad(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
context.EmitLdint(op.Rn);
context.EmitSttmp();
if (exclusive)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.SetExclusiveAddress));
}
void WriteExclusiveValue(string propName)
{
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U8);
}
context.EmitSttmp2();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp2();
context.EmitCallPrivatePropSet(typeof(CpuThreadState), propName);
context.EmitLdtmp2();
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
}
if (pair)
{
//Exclusive loads should be atomic. For pairwise loads, we need to
//read all the data at once. For a 32-bits pairwise load, we do a
//simple 64-bits load, for a 128-bits load, we need to call a special
//method to read 128-bits atomically.
if (op.Size == 2)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, 3);
context.Emit(OpCodes.Dup);
//Mask low half.
context.Emit(OpCodes.Conv_U4);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
//Shift high half.
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh));
}
context.EmitStintzr(op.Rt2);
}
else if (op.Size == 3)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicReadInt128));
context.Emit(OpCodes.Dup);
//Load low part of the vector.
context.EmitLdc_I4(0);
context.EmitLdc_I4(3);
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx));
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
//Load high part of the vector.
context.EmitLdc_I4(1);
context.EmitLdc_I4(3);
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx));
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh));
}
context.EmitStintzr(op.Rt2);
}
else
{
throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
else
{
//8, 16, 32 or 64-bits (non-pairwise) load.
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, op.Size);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
}
}