private static void EmitStore(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
ILLabel lblEx = new ILLabel();
ILLabel lblEnd = new ILLabel();
if (exclusive)
{
EmitMemoryCall(context, nameof(MemoryManager.TestExclusive), op.Rn);
context.Emit(OpCodes.Brtrue_S, lblEx);
context.EmitLdc_I8(1);
context.EmitStintzr(op.Rs);
context.Emit(OpCodes.Br_S, lblEnd);
}
context.MarkLabel(lblEx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdintzr(op.Rt);
EmitWriteCall(context, op.Size);
if (pair)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
context.EmitLdintzr(op.Rt2);
EmitWriteCall(context, op.Size);
}
if (exclusive)
{
context.EmitLdc_I8(0);
context.EmitStintzr(op.Rs);
EmitMemoryCall(context, nameof(MemoryManager.ClearExclusiveForStore));
}
context.MarkLabel(lblEnd);
}
public static void Bfm(ILEmitterCtx context)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
if (op.Pos < op.Shift)
{
//BFI.
context.EmitLdintzr(op.Rn);
int shift = op.GetBitsCount() - op.Shift;
int width = op.Pos + 1;
long mask = (long)(ulong.MaxValue >> (64 - width));
context.EmitLdc_I(mask);
context.Emit(OpCodes.And);
context.EmitLsl(shift);
context.EmitLdintzr(op.Rd);
context.EmitLdc_I(~(mask << shift));
context.Emit(OpCodes.And);
context.Emit(OpCodes.Or);
context.EmitStintzr(op.Rd);
}
else
{
//BFXIL.
context.EmitLdintzr(op.Rn);
context.EmitLsr(op.Shift);
int width = op.Pos - op.Shift + 1;
long mask = (long)(ulong.MaxValue >> (64 - width));
context.EmitLdc_I(mask);
context.Emit(OpCodes.And);
context.EmitLdintzr(op.Rd);
context.EmitLdc_I(~mask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Or);
context.EmitStintzr(op.Rd);
}
}
private static void EmitCsel(ILEmitterCtx context, CselOperation cselOp)
{
OpCodeCsel64 op = (OpCodeCsel64)context.CurrOp;
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitCondBranch(lblTrue, op.Cond);
context.EmitLdintzr(op.Rm);
if (cselOp == CselOperation.Increment)
{
context.EmitLdc_I(1);
context.Emit(OpCodes.Add);
}
else if (cselOp == CselOperation.Invert)
{
context.Emit(OpCodes.Not);
}
else if (cselOp == CselOperation.Negate)
{
context.Emit(OpCodes.Neg);
}
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
context.EmitLdintzr(op.Rn);
context.MarkLabel(lblEnd);
context.EmitStintzr(op.Rd);
}
public static void Adrp(ILEmitterCtx context)
{
OpCodeAdr64 op = (OpCodeAdr64)context.CurrOp;
context.EmitLdc_I((op.Position & ~0xfffL) + (op.Imm << 12));
context.EmitStintzr(op.Rd);
}
private static void EmitLdr(ILEmitterCtx context, bool signed)
{
OpCodeMem64 op = (OpCodeMem64)context.CurrOp;
EmitLoadAddress(context);
if (signed && op.Extend64)
{
EmitReadSx64Call(context, op.Size);
}
else if (signed)
{
EmitReadSx32Call(context, op.Size);
}
else
{
EmitReadZxCall(context, op.Size);
}
if (op is IOpCodeSimd64)
{
context.EmitStvec(op.Rt);
}
else
{
context.EmitStintzr(op.Rt);
}
EmitWBackIfNeeded(context);
}
public static void Ubfm(ILEmitterCtx context)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
if (op.Pos + 1 == op.GetBitsCount())
{
EmitUbfmShift(context);
}
else if (op.Pos < op.Shift)
{
EmitUbfiz(context);
}
else if (op.Pos + 1 == op.Shift)
{
EmitBfmLsl(context);
}
else if (op.Pos == 7 && op.Shift == 0)
{
EmitUbfmCast(context, OpCodes.Conv_U1);
}
else if (op.Pos == 15 && op.Shift == 0)
{
EmitUbfmCast(context, OpCodes.Conv_U2);
}
else
{
EmitBfmLoadRn(context);
context.EmitStintzr(op.Rd);
}
}
public static void Adr(ILEmitterCtx context)
{
OpCodeAdr64 op = (OpCodeAdr64)context.CurrOp;
context.EmitLdc_I(op.Position + op.Imm);
context.EmitStintzr(op.Rd);
}
public static void Movz(ILEmitterCtx context)
{
OpCodeMov64 op = (OpCodeMov64)context.CurrOp;
context.EmitLdc_I(op.Imm);
context.EmitStintzr(op.Rd);
}
public static void Fmov_Ftoi1(ILEmitterCtx context)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
EmitVectorExtractZx(context, op.Rn, 1, 3);
context.EmitStintzr(op.Rd);
}
public static void Umov_S(ILEmitterCtx context)
{
OpCodeSimdIns64 op = (OpCodeSimdIns64)context.CurrOp;
EmitVectorExtractZx(context, op.Rn, op.DstIndex, op.Size);
context.EmitStintzr(op.Rd);
}
public static void Smov_S(ILEmitterCtx context)
{
OpCodeSimdIns64 op = (OpCodeSimdIns64)context.CurrOp;
EmitVectorExtractSx(context, op.Rn, op.DstIndex, op.Size);
EmitIntZeroUpperIfNeeded(context);
context.EmitStintzr(op.Rd);
}
public static void Rev64(ILEmitterCtx context)
{
OpCodeAlu64 op = (OpCodeAlu64)context.CurrOp;
context.EmitLdintzr(op.Rn);
SoftFallback.EmitCall(context, nameof(SoftFallback.ReverseBytes64));
context.EmitStintzr(op.Rd);
}
public static void Fmov_Ftoi1(ILEmitterCtx context)
{
OpCodeSimdCvt64 op = (OpCodeSimdCvt64)context.CurrOp;
EmitVectorExtractZx(context, op.Rn, 1, 3);
EmitIntZeroUpperIfNeeded(context);
context.EmitStintzr(op.Rd);
}
private static void EmitBfmLsl(ILEmitterCtx context)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitLsl(op.GetBitsCount() - op.Shift);
context.EmitStintzr(op.Rd);
}
private static void EmitSse42Crc32(ILEmitterCtx context, Type tCrc, Type tData)
{
OpCodeAluRs64 op = (OpCodeAluRs64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitLdintzr(op.Rm);
context.EmitCall(typeof(Sse42).GetMethod(nameof(Sse42.Crc32), new Type[] { tCrc, tData }));
context.EmitStintzr(op.Rd);
}
public static void Umulh(ILEmitterCtx context)
{
OpCodeMul64 op = (OpCodeMul64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitLdintzr(op.Rm);
SoftFallback.EmitCall(context, nameof(SoftFallback.UMulHi128));
context.EmitStintzr(op.Rd);
}
public static void Cls(ILEmitterCtx context)
{
OpCodeAlu64 op = (OpCodeAlu64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitLdc_I4(op.RegisterSize == RegisterSize.Int32 ? 32 : 64);
SoftFallback.EmitCall(context, nameof(SoftFallback.CountLeadingSigns));
context.EmitStintzr(op.Rd);
}
private static void EmitLoad(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
if (exclusive)
{
EmitMemoryCall(context, nameof(MemoryManager.SetExclusive), op.Rn);
}
context.EmitLdint(op.Rn);
context.EmitSttmp();
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, op.Size);
context.EmitStintzr(op.Rt);
if (pair)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
EmitReadZxCall(context, op.Size);
context.EmitStintzr(op.Rt2);
}
}
private static void EmitBfmShift(ILEmitterCtx context, bool signed)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.EmitLdc_I4(op.Shift);
context.Emit(signed
? OpCodes.Shr
: OpCodes.Shr_Un);
context.EmitStintzr(op.Rd);
}
public static void EmitAluStore(ILEmitterCtx context, bool setFlags)
{
IOpCodeAlu64 op = (IOpCodeAlu64)context.CurrOp;
if (setFlags || op is IOpCodeAluRs64)
{
context.EmitStintzr(op.Rd);
}
else
{
context.EmitStint(op.Rd);
}
}
private static void EmitMul(ILEmitterCtx context, OpCode ilOp)
{
OpCodeMul64 op = (OpCodeMul64)context.CurrOp;
context.EmitLdintzr(op.Ra);
context.EmitLdintzr(op.Rn);
context.EmitLdintzr(op.Rm);
context.Emit(OpCodes.Mul);
context.Emit(ilOp);
context.EmitStintzr(op.Rd);
}
public static void Smov_S(ILEmitterCtx context)
{
OpCodeSimdIns64 op = (OpCodeSimdIns64)context.CurrOp;
EmitVectorExtractSx(context, op.Rn, op.DstIndex, op.Size);
if (op.RegisterSize == RegisterSize.Simd64)
{
context.Emit(OpCodes.Conv_U4);
context.Emit(OpCodes.Conv_U8);
}
context.EmitStintzr(op.Rd);
}
public static void Movk(ILEmitterCtx context)
{
OpCodeMov64 op = (OpCodeMov64)context.CurrOp;
context.EmitLdintzr(op.Rd);
context.EmitLdc_I(~(0xffffL << op.Pos));
context.Emit(OpCodes.And);
context.EmitLdc_I(op.Imm);
context.Emit(OpCodes.Or);
context.EmitStintzr(op.Rd);
}
private static void EmitFallback32_64(ILEmitterCtx context, string name32, string name64)
{
OpCodeAlu64 op = (OpCodeAlu64)context.CurrOp;
context.EmitLdintzr(op.Rn);
if (op.RegisterSize == RegisterSize.Int32)
{
SoftFallback.EmitCall(context, name32);
}
else
{
SoftFallback.EmitCall(context, name64);
}
context.EmitStintzr(op.Rd);
}
private static void EmitBfmCast(ILEmitterCtx context, OpCode ilOp, bool signed)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
context.EmitLdintzr(op.Rn);
context.Emit(ilOp);
if (op.RegisterSize != RegisterSize.Int32)
{
context.Emit(signed
? OpCodes.Conv_I8
: OpCodes.Conv_U8);
}
context.EmitStintzr(op.Rd);
}
public static void Sbfm(ILEmitterCtx context)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
int bitsCount = op.GetBitsCount();
if (op.Pos + 1 == bitsCount)
{
EmitSbfmShift(context);
}
else if (op.Pos < op.Shift)
{
EmitSbfiz(context);
}
else if (op.Pos == 7 && op.Shift == 0)
{
EmitSbfmCast(context, OpCodes.Conv_I1);
}
else if (op.Pos == 15 && op.Shift == 0)
{
EmitSbfmCast(context, OpCodes.Conv_I2);
}
else if (op.Pos == 31 && op.Shift == 0)
{
EmitSbfmCast(context, OpCodes.Conv_I4);
}
else
{
EmitBfmLoadRn(context);
context.EmitLdintzr(op.Rn);
context.EmitLsl(bitsCount - 1 - op.Pos);
context.EmitAsr(bitsCount - 1);
context.EmitLdc_I(~op.TMask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Or);
context.EmitStintzr(op.Rd);
}
}
public static void Bfm(ILEmitterCtx context)
{
OpCodeBfm64 op = (OpCodeBfm64)context.CurrOp;
EmitBfmLoadRn(context);
context.EmitLdintzr(op.Rd);
context.EmitLdc_I(~op.WMask & op.TMask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Or);
context.EmitLdintzr(op.Rd);
context.EmitLdc_I(~op.TMask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Or);
context.EmitStintzr(op.Rd);
}
public static void Mrs(ILEmitterCtx context)
{
OpCodeSystem64 op = (OpCodeSystem64)context.CurrOp;
context.EmitLdarg(TranslatedSub.StateArgIdx);
string propName;
switch (GetPackedId(op))
{
case 0b11_011_0000_0000_001: propName = nameof(CpuThreadState.CtrEl0); break;
case 0b11_011_0000_0000_111: propName = nameof(CpuThreadState.DczidEl0); break;
case 0b11_011_0100_0100_000: propName = nameof(CpuThreadState.Fpcr); break;
case 0b11_011_0100_0100_001: propName = nameof(CpuThreadState.Fpsr); break;
case 0b11_011_1101_0000_010: propName = nameof(CpuThreadState.TpidrEl0); break;
case 0b11_011_1101_0000_011: propName = nameof(CpuThreadState.Tpidr); break;
case 0b11_011_1110_0000_000: propName = nameof(CpuThreadState.CntfrqEl0); break;
case 0b11_011_1110_0000_001: propName = nameof(CpuThreadState.CntpctEl0); break;
default: throw new NotImplementedException($"Unknown MRS at {op.Position:x16}");
}
context.EmitCallPropGet(typeof(CpuThreadState), propName);
PropertyInfo propInfo = typeof(CpuThreadState).GetProperty(propName);
if (propInfo.PropertyType != typeof(long) &&
propInfo.PropertyType != typeof(ulong))
{
context.Emit(OpCodes.Conv_U8);
}
context.EmitStintzr(op.Rt);
}
public static void Ldp(ILEmitterCtx context)
{
OpCodeMemPair64 op = (OpCodeMemPair64)context.CurrOp;
void EmitReadAndStore(int rt)
{
if (op.Extend64)
{
EmitReadSx64Call(context, op.Size);
}
else
{
EmitReadZxCall(context, op.Size);
}
if (op is IOpCodeSimd64)
{
context.EmitStvec(rt);
}
else
{
context.EmitStintzr(rt);
}
}
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
EmitReadAndStore(op.Rt);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
EmitReadAndStore(op.Rt2);
EmitWBackIfNeeded(context);
}
public static void Clz(ILEmitterCtx context)
{
OpCodeAlu64 op = (OpCodeAlu64)context.CurrOp;
context.EmitLdintzr(op.Rn);
if (Lzcnt.IsSupported)
{
Type tValue = op.RegisterSize == RegisterSize.Int32 ? typeof(uint) : typeof(ulong);
context.EmitCall(typeof(Lzcnt).GetMethod(nameof(Lzcnt.LeadingZeroCount), new Type[] { tValue }));
}
else
{
context.EmitLdc_I4(op.RegisterSize == RegisterSize.Int32 ? 32 : 64);
SoftFallback.EmitCall(context, nameof(SoftFallback.CountLeadingZeros));
}
context.EmitStintzr(op.Rd);
}