public static void EmitBxWritePc(ILEmitterCtx context)
{
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(1);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Dup);
context.EmitStflg((int)PState.TBit);
ILLabel lblArmMode = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblArmMode);
context.EmitLdc_I4(~1);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblArmMode);
context.EmitLdc_I4(~3);
context.MarkLabel(lblEnd);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Conv_U8);
context.Emit(OpCodes.Ret);
}
private static void EmitCsel(ILEmitterCtx context, CselOperation cselOp)
{
OpCodeCsel64 op = (OpCodeCsel64)context.CurrOp;
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitCondBranch(lblTrue, op.Cond);
context.EmitLdintzr(op.Rm);
if (cselOp == CselOperation.Increment)
{
context.EmitLdc_I(1);
context.Emit(OpCodes.Add);
}
else if (cselOp == CselOperation.Invert)
{
context.Emit(OpCodes.Not);
}
else if (cselOp == CselOperation.Negate)
{
context.Emit(OpCodes.Neg);
}
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
context.EmitLdintzr(op.Rn);
context.MarkLabel(lblEnd);
context.EmitStintzr(op.Rd);
}
private static void EmitReadInt(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitReadIntFallback(context, size);
context.Emit(OpCodes.Br, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
switch (size)
{
case 0: context.Emit(OpCodes.Ldind_U1); break;
case 1: context.Emit(OpCodes.Ldind_U2); break;
case 2: context.Emit(OpCodes.Ldind_U4); break;
case 3: context.Emit(OpCodes.Ldind_I8); break;
}
context.MarkLabel(lblEnd);
}
private static void EmitWriteVector(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitWriteVectorFallback(context, size);
context.Emit(OpCodes.Br, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
context.EmitLdvec(_tempVecValue);
switch (size)
{
case 2: context.EmitCall(typeof(Sse), nameof(Sse.StoreScalar)); break;
case 3: context.EmitCall(typeof(Sse2), nameof(Sse2.StoreScalar)); break;
case 4: context.EmitCall(typeof(Sse), nameof(Sse.StoreAligned)); break;
default: throw new InvalidOperationException($"Invalid vector store size of {1 << size} bytes.");
}
context.MarkLabel(lblEnd);
}
private static void EmitStore(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
ILLabel lblEx = new ILLabel();
ILLabel lblEnd = new ILLabel();
if (exclusive)
{
EmitMemoryCall(context, nameof(MemoryManager.TestExclusive), op.Rn);
context.Emit(OpCodes.Brtrue_S, lblEx);
context.EmitLdc_I8(1);
context.EmitStintzr(op.Rs);
context.Emit(OpCodes.Br_S, lblEnd);
}
context.MarkLabel(lblEx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdintzr(op.Rt);
EmitWriteCall(context, op.Size);
if (pair)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
context.EmitLdintzr(op.Rt2);
EmitWriteCall(context, op.Size);
}
if (exclusive)
{
context.EmitLdc_I8(0);
context.EmitStintzr(op.Rs);
EmitMemoryCall(context, nameof(MemoryManager.ClearExclusiveForStore));
}
context.MarkLabel(lblEnd);
}
private static void EmitCcmp(ILEmitterCtx context, CcmpOp cmpOp)
{
OpCodeCcmp64 op = (OpCodeCcmp64)context.CurrOp;
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitCondBranch(lblTrue, op.Cond);
context.EmitLdc_I4((op.Nzcv >> 0) & 1);
context.EmitStflg((int)PState.VBit);
context.EmitLdc_I4((op.Nzcv >> 1) & 1);
context.EmitStflg((int)PState.CBit);
context.EmitLdc_I4((op.Nzcv >> 2) & 1);
context.EmitStflg((int)PState.ZBit);
context.EmitLdc_I4((op.Nzcv >> 3) & 1);
context.EmitStflg((int)PState.NBit);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
EmitAluLoadOpers(context);
if (cmpOp == CcmpOp.Cmp)
{
context.Emit(OpCodes.Sub);
context.EmitZnFlagCheck();
EmitSubsCCheck(context);
EmitSubsVCheck(context);
}
else if (cmpOp == CcmpOp.Cmn)
{
context.Emit(OpCodes.Add);
context.EmitZnFlagCheck();
EmitAddsCCheck(context);
EmitAddsVCheck(context);
}
else
{
throw new ArgumentException(nameof(cmpOp));
}
context.Emit(OpCodes.Pop);
context.MarkLabel(lblEnd);
}
private static void EmitFcmp(ILEmitterCtx context, OpCode ilOp, int index, bool scalar)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
int sizeF = op.Size & 1;
ulong szMask = ulong.MaxValue >> (64 - (32 << sizeF));
EmitVectorExtractF(context, op.Rn, index, sizeF);
if (op is OpCodeSimdReg64 binOp)
{
EmitVectorExtractF(context, binOp.Rm, index, sizeF);
}
else if (sizeF == 0)
{
context.EmitLdc_R4(0f);
}
else /* if (SizeF == 1) */
{
context.EmitLdc_R8(0d);
}
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(ilOp, lblTrue);
if (scalar)
{
EmitVectorZeroAll(context, op.Rd);
}
else
{
EmitVectorInsert(context, op.Rd, index, sizeF + 2, 0);
}
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
if (scalar)
{
EmitVectorInsert(context, op.Rd, index, 3, (long)szMask);
EmitVectorZeroUpper(context, op.Rd);
}
else
{
EmitVectorInsert(context, op.Rd, index, sizeF + 2, (long)szMask);
}
context.MarkLabel(lblEnd);
}
private static void EmitContinueOrReturnCheck(ILEmitterCtx context)
{
// Note: The return value of the called method will be placed
// at the Stack, the return value is always a Int64 with the
// return address of the function. We check if the address is
// correct, if it isn't we keep returning until we reach the dispatcher.
if (context.CurrBlock.Next != null)
{
context.Emit(OpCodes.Dup);
context.EmitLdc_I8(context.CurrOp.Position + 4);
ILLabel lblContinue = new ILLabel();
context.Emit(OpCodes.Beq_S, lblContinue);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblContinue);
context.Emit(OpCodes.Pop);
context.EmitLoadContext();
}
else
{
context.Emit(OpCodes.Ret);
}
}
private static void EmitBranch(ILEmitterCtx context, Condition cond)
{
OpCodeBImm64 op = (OpCodeBImm64)context.CurrOp;
if (context.CurrBlock.Branch != null)
{
context.EmitCondBranch(context.GetLabel(op.Imm), cond);
if (context.CurrBlock.Next == null)
{
context.EmitStoreContext();
context.EmitLdc_I8(op.Position + 4);
context.Emit(OpCodes.Ret);
}
}
else
{
context.EmitStoreContext();
ILLabel lblTaken = new ILLabel();
context.EmitCondBranch(lblTaken, cond);
context.EmitLdc_I8(op.Position + 4);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblTaken);
context.EmitLdc_I8(op.Imm);
context.Emit(OpCodes.Ret);
}
}
public static void EmitCall(ILEmitterCtx context, long imm)
{
if (context.TryOptEmitSubroutineCall())
{
//Note: the return value of the called method will be placed
//at the Stack, the return value is always a Int64 with the
//return address of the function. We check if the address is
//correct, if it isn't we keep returning until we reach the dispatcher.
context.Emit(OpCodes.Dup);
context.EmitLdc_I8(context.CurrOp.Position + 4);
ILLabel lblContinue = new ILLabel();
context.Emit(OpCodes.Beq_S, lblContinue);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblContinue);
context.Emit(OpCodes.Pop);
context.EmitLoadState();
}
else
{
context.EmitLdc_I8(imm);
context.Emit(OpCodes.Ret);
}
}
private static void EmitBranch(ILEmitterCtx context, OpCode ilOp)
{
OpCodeBImm64 op = (OpCodeBImm64)context.CurrOp;
if (context.CurrBlock.Next != null &&
context.CurrBlock.Branch != null)
{
context.Emit(ilOp, context.GetLabel(op.Imm));
}
else
{
context.EmitStoreState();
ILLabel lblTaken = new ILLabel();
context.Emit(ilOp, lblTaken);
context.EmitLdc_I8(op.Position + 4);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblTaken);
context.EmitLdc_I8(op.Imm);
context.Emit(OpCodes.Ret);
}
}
public static void Fccmp_S(ILEmitterCtx context)
{
OpCodeSimdFcond64 op = (OpCodeSimdFcond64)context.CurrOp;
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitCondBranch(lblTrue, op.Cond);
EmitSetNzcv(context, op.Nzcv);
context.Emit(OpCodes.Br, lblEnd);
context.MarkLabel(lblTrue);
Fcmp_S(context);
context.MarkLabel(lblEnd);
}
private static void EmitCmpOp(ILEmitterCtx context, OpCode ilOp, bool scalar)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
int bytes = op.GetBitsCount() >> 3;
int elems = !scalar ? bytes >> op.Size : 1;
ulong szMask = ulong.MaxValue >> (64 - (8 << op.Size));
for (int index = 0; index < elems; index++)
{
EmitVectorExtractSx(context, op.Rn, index, op.Size);
if (op is OpCodeSimdReg64 binOp)
{
EmitVectorExtractSx(context, binOp.Rm, index, op.Size);
}
else
{
context.EmitLdc_I8(0L);
}
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(ilOp, lblTrue);
EmitVectorInsert(context, op.Rd, index, op.Size, 0);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
EmitVectorInsert(context, op.Rd, index, op.Size, (long)szMask);
context.MarkLabel(lblEnd);
}
if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
private static void EmitReadVector(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitReadVectorFallback(context, size);
context.Emit(OpCodes.Br, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
switch (size)
{
case 2: context.EmitCall(typeof(Sse), nameof(Sse.LoadScalarVector128)); break;
case 3:
{
Type[] types = new Type[] { typeof(double *) };
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.LoadScalarVector128), types));
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorDoubleToSingle));
break;
}
case 4: context.EmitCall(typeof(Sse), nameof(Sse.LoadAlignedVector128)); break;
throw new InvalidOperationException($"Invalid vector load size of {1 << size} bytes.");
}
context.MarkLabel(lblEnd);
}
private static void EmitAluStore(ILEmitterCtx context)
{
IOpCode32Alu op = (IOpCode32Alu)context.CurrOp;
if (op.Rd == RegisterAlias.Aarch32Pc)
{
if (op.SetFlags)
{
// TODO: Load SPSR etc.
context.EmitLdflg((int)PState.TBit);
ILLabel lblThumb = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblThumb);
context.EmitLdc_I4(~3);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblThumb);
context.EmitLdc_I4(~1);
context.MarkLabel(lblEnd);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Conv_U8);
context.Emit(OpCodes.Ret);
}
else
{
EmitAluWritePc(context);
}
}
else
{
context.EmitStint(GetRegisterAlias(context.Mode, op.Rd));
}
}
public static void Fcsel_S(ILEmitterCtx context)
{
OpCodeSimdFcond64 op = (OpCodeSimdFcond64)context.CurrOp;
ILLabel lblTrue = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitCondBranch(lblTrue, op.Cond);
EmitVectorExtractF(context, op.Rm, 0, op.Size);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblTrue);
EmitVectorExtractF(context, op.Rn, 0, op.Size);
context.MarkLabel(lblEnd);
EmitScalarSetF(context, op.Rd, op.Size);
}
private static void EmitWriteInt(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitWriteIntFallback(context, size);
context.Emit(OpCodes.Br, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
context.EmitLdint(_tempIntValue);
if (size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
switch (size)
{
case 0: context.Emit(OpCodes.Stind_I1); break;
case 1: context.Emit(OpCodes.Stind_I2); break;
case 2: context.Emit(OpCodes.Stind_I4); break;
case 3: context.Emit(OpCodes.Stind_I8); break;
}
context.MarkLabel(lblEnd);
}
private static void EmitDiv(ILEmitterCtx context, OpCode ilOp)
{
// If Rm == 0, Rd = 0 (division by zero).
context.EmitLdc_I(0);
EmitAluLoadRm(context);
context.EmitLdc_I(0);
ILLabel badDiv = new ILLabel();
context.Emit(OpCodes.Beq_S, badDiv);
context.Emit(OpCodes.Pop);
if (ilOp == OpCodes.Div)
{
// If Rn == INT_MIN && Rm == -1, Rd = INT_MIN (overflow).
long intMin = 1L << (context.CurrOp.GetBitsCount() - 1);
context.EmitLdc_I(intMin);
EmitAluLoadRn(context);
context.EmitLdc_I(intMin);
context.Emit(OpCodes.Ceq);
EmitAluLoadRm(context);
context.EmitLdc_I(-1);
context.Emit(OpCodes.Ceq);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Brtrue_S, badDiv);
context.Emit(OpCodes.Pop);
}
EmitAluLoadRn(context);
EmitAluLoadRm(context);
context.Emit(ilOp);
context.MarkLabel(badDiv);
EmitAluStore(context);
}
private static void EmitExceptionCall(ILEmitterCtx context, string mthdName)
{
OpCodeException64 op = (OpCodeException64)context.CurrOp;
context.EmitStoreState();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdc_I8(op.Position);
context.EmitLdc_I4(op.Id);
context.EmitPrivateCall(typeof(CpuThreadState), mthdName);
//Check if the thread should still be running, if it isn't then we return 0
//to force a return to the dispatcher and then exit the thread.
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCallPropGet(typeof(CpuThreadState), nameof(CpuThreadState.Running));
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblEnd);
context.EmitLdc_I8(0);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblEnd);
if (context.CurrBlock.Next != null)
{
context.EmitLoadState(context.CurrBlock.Next);
}
else
{
context.EmitLdc_I8(op.Position + 4);
context.Emit(OpCodes.Ret);
}
}
public static void Bl(ILEmitterCtx context)
{
OpCodeBImmAl64 op = (OpCodeBImmAl64)context.CurrOp;
context.EmitLdc_I(op.Position + 4);
context.EmitStint(CpuThreadState.LrIndex);
context.EmitStoreState();
if (context.TryOptEmitSubroutineCall())
{
//Note: the return value of the called method will be placed
//at the Stack, the return value is always a Int64 with the
//return address of the function. We check if the address is
//correct, if it isn't we keep returning until we reach the dispatcher.
context.Emit(OpCodes.Dup);
context.EmitLdc_I8(op.Position + 4);
ILLabel lblContinue = new ILLabel();
context.Emit(OpCodes.Beq_S, lblContinue);
context.Emit(OpCodes.Ret);
context.MarkLabel(lblContinue);
context.Emit(OpCodes.Pop);
context.EmitLoadState(context.CurrBlock.Next);
}
else
{
context.EmitLdc_I8(op.Imm);
context.Emit(OpCodes.Ret);
}
}
public static void Fcmp_S(ILEmitterCtx context)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
bool cmpWithZero = !(op is OpCodeSimdFcond64) ? op.Bit3 : false;
//Handle NaN case.
//If any number is NaN, then NZCV = 0011.
if (cmpWithZero)
{
EmitNaNCheck(context, op.Rn);
}
else
{
EmitNaNCheck(context, op.Rn);
EmitNaNCheck(context, op.Rm);
context.Emit(OpCodes.Or);
}
ILLabel lblNaN = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblNaN);
void EmitLoadOpers()
{
EmitVectorExtractF(context, op.Rn, 0, op.Size);
if (cmpWithZero)
{
if (op.Size == 0)
{
context.EmitLdc_R4(0f);
}
else /* if (Op.Size == 1) */
{
context.EmitLdc_R8(0d);
}
}
else
{
EmitVectorExtractF(context, op.Rm, 0, op.Size);
}
}
//Z = Rn == Rm
EmitLoadOpers();
context.Emit(OpCodes.Ceq);
context.Emit(OpCodes.Dup);
context.EmitStflg((int)PState.ZBit);
//C = Rn >= Rm
EmitLoadOpers();
context.Emit(OpCodes.Cgt);
context.Emit(OpCodes.Or);
context.EmitStflg((int)PState.CBit);
//N = Rn < Rm
EmitLoadOpers();
context.Emit(OpCodes.Clt);
context.EmitStflg((int)PState.NBit);
//V = 0
context.EmitLdc_I4(0);
context.EmitStflg((int)PState.VBit);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblNaN);
EmitSetNzcv(context, 0b0011);
context.MarkLabel(lblEnd);
}
private static void EmitVectorShl(ILEmitterCtx context, bool signed)
{
//This instruction shifts the value on vector A by the number of bits
//specified on the signed, lower 8 bits of vector B. If the shift value
//is greater or equal to the data size of each lane, then the result is zero.
//Additionally, negative shifts produces right shifts by the negated shift value.
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
int maxShift = 8 << op.Size;
Action emit = () =>
{
ILLabel lblShl = new ILLabel();
ILLabel lblZero = new ILLabel();
ILLabel lblEnd = new ILLabel();
void EmitShift(OpCode ilOp)
{
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(maxShift);
context.Emit(OpCodes.Bge_S, lblZero);
context.Emit(ilOp);
context.Emit(OpCodes.Br_S, lblEnd);
}
context.Emit(OpCodes.Conv_I1);
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(0);
context.Emit(OpCodes.Bge_S, lblShl);
context.Emit(OpCodes.Neg);
EmitShift(signed
? OpCodes.Shr
: OpCodes.Shr_Un);
context.MarkLabel(lblShl);
EmitShift(OpCodes.Shl);
context.MarkLabel(lblZero);
context.Emit(OpCodes.Pop);
context.Emit(OpCodes.Pop);
context.EmitLdc_I8(0);
context.MarkLabel(lblEnd);
};
if (signed)
{
EmitVectorBinaryOpSx(context, emit);
}
else
{
EmitVectorBinaryOpZx(context, emit);
}
}
private static void EmitFcmpOrFcmpe(ILEmitterCtx context, bool signalNaNs)
{
OpCodeSimdReg64 op = (OpCodeSimdReg64)context.CurrOp;
bool cmpWithZero = !(op is OpCodeSimdFcond64) ? op.Bit3 : false;
if (Optimizations.FastFP && Optimizations.UseSse2)
{
if (op.Size == 0)
{
Type[] typesCmp = new Type[] { typeof(Vector128 <float>), typeof(Vector128 <float>) };
ILLabel lblNaN = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitLdvec(op.Rn);
context.Emit(OpCodes.Dup);
context.EmitStvectmp();
if (cmpWithZero)
{
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorSingleZero));
}
else
{
context.EmitLdvec(op.Rm);
}
context.Emit(OpCodes.Dup);
context.EmitStvectmp2();
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.CompareOrderedScalar), typesCmp));
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorSingleZero));
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.CompareEqualOrderedScalar), typesCmp));
context.Emit(OpCodes.Brtrue_S, lblNaN);
context.EmitLdc_I4(0);
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.CompareGreaterThanOrEqualOrderedScalar), typesCmp));
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.CompareEqualOrderedScalar), typesCmp));
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.CompareLessThanOrderedScalar), typesCmp));
context.EmitStflg((int)PState.NBit);
context.EmitStflg((int)PState.ZBit);
context.EmitStflg((int)PState.CBit);
context.EmitStflg((int)PState.VBit);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblNaN);
context.EmitLdc_I4(1);
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(0);
context.Emit(OpCodes.Dup);
context.EmitStflg((int)PState.NBit);
context.EmitStflg((int)PState.ZBit);
context.EmitStflg((int)PState.CBit);
context.EmitStflg((int)PState.VBit);
context.MarkLabel(lblEnd);
}
else /* if (op.Size == 1) */
{
Type[] typesCmp = new Type[] { typeof(Vector128 <double>), typeof(Vector128 <double>) };
ILLabel lblNaN = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitLdvec(op.Rn);
context.Emit(OpCodes.Dup);
context.EmitStvectmp();
if (cmpWithZero)
{
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorDoubleZero));
}
else
{
context.EmitLdvec(op.Rm);
}
context.Emit(OpCodes.Dup);
context.EmitStvectmp2();
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.CompareOrderedScalar), typesCmp));
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorDoubleZero));
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.CompareEqualOrderedScalar), typesCmp));
context.Emit(OpCodes.Brtrue_S, lblNaN);
context.EmitLdc_I4(0);
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.CompareGreaterThanOrEqualOrderedScalar), typesCmp));
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.CompareEqualOrderedScalar), typesCmp));
context.EmitLdvectmp();
context.EmitLdvectmp2();
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.CompareLessThanOrderedScalar), typesCmp));
context.EmitStflg((int)PState.NBit);
context.EmitStflg((int)PState.ZBit);
context.EmitStflg((int)PState.CBit);
context.EmitStflg((int)PState.VBit);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblNaN);
context.EmitLdc_I4(1);
context.Emit(OpCodes.Dup);
context.EmitLdc_I4(0);
context.Emit(OpCodes.Dup);
context.EmitStflg((int)PState.NBit);
context.EmitStflg((int)PState.ZBit);
context.EmitStflg((int)PState.CBit);
context.EmitStflg((int)PState.VBit);
context.MarkLabel(lblEnd);
}
}
else
{
EmitVectorExtractF(context, op.Rn, 0, op.Size);
if (cmpWithZero)
{
if (op.Size == 0)
{
context.EmitLdc_R4(0f);
}
else /* if (op.Size == 1) */
{
context.EmitLdc_R8(0d);
}
}
else
{
EmitVectorExtractF(context, op.Rm, 0, op.Size);
}
context.EmitLdc_I4(!signalNaNs ? 0 : 1);
EmitSoftFloatCall(context, nameof(SoftFloat32.FPCompare));
EmitSetNzcv(context);
}
}
private static void EmitLoadOrStore(ILEmitterCtx context, int size, AccessType accType)
{
OpCode32Mem op = (OpCode32Mem)context.CurrOp;
if (op.Index || op.WBack)
{
EmitLoadFromRegister(context, op.Rn);
context.EmitLdc_I4(op.Imm);
context.Emit(op.Add ? OpCodes.Add : OpCodes.Sub);
context.EmitSttmp();
}
if (op.Index)
{
context.EmitLdtmp();
}
else
{
EmitLoadFromRegister(context, op.Rn);
}
if ((accType & AccessType.Load) != 0)
{
if ((accType & AccessType.Signed) != 0)
{
EmitReadSx32Call(context, size);
}
else
{
EmitReadZxCall(context, size);
}
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Dup);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.MarkLabel(lblEnd);
}
else
{
EmitStoreToRegister(context, op.Rt);
}
}
else
{
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
EmitLoadFromRegister(context, op.Rt);
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Conv_U8);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.EmitLsl(32);
context.Emit(OpCodes.Or);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsl(32);
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.Emit(OpCodes.Or);
context.MarkLabel(lblEnd);
}
EmitWriteCall(context, size);
}
}
private static void EmitStore(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
if (exclusive)
{
ILLabel lblEx = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdint(op.Rn);
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.CheckExclusiveAddress));
context.Emit(OpCodes.Brtrue_S, lblEx);
//Address check failed, set error right away and do not store anything.
context.EmitLdc_I4(1);
context.EmitStintzr(op.Rs);
context.Emit(OpCodes.Br, lblEnd);
//Address check passsed.
context.MarkLabel(lblEx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueLow));
void EmitCast()
{
//The input should be always int64.
switch (op.Size)
{
case 0: context.Emit(OpCodes.Conv_U1); break;
case 1: context.Emit(OpCodes.Conv_U2); break;
case 2: context.Emit(OpCodes.Conv_U4); break;
}
}
EmitCast();
if (pair)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueHigh));
EmitCast();
context.EmitLdintzr(op.Rt);
EmitCast();
context.EmitLdintzr(op.Rt2);
EmitCast();
switch (op.Size)
{
case 2: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchange2xInt32)); break;
case 3: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt128)); break;
default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
else
{
context.EmitLdintzr(op.Rt);
EmitCast();
switch (op.Size)
{
case 0: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeByte)); break;
case 1: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt16)); break;
case 2: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt32)); break;
case 3: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt64)); break;
default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
//The value returned is a bool, true if the values compared
//were equal and the new value was written, false otherwise.
//We need to invert this result, as on ARM 1 indicates failure,
//and 0 success on those instructions.
context.EmitLdc_I4(1);
context.Emit(OpCodes.Xor);
context.Emit(OpCodes.Dup);
context.Emit(OpCodes.Conv_U8);
context.EmitStintzr(op.Rs);
//Only clear the exclusive monitor if the store was successful (Rs = false).
context.Emit(OpCodes.Brtrue_S, lblEnd);
Clrex(context);
context.MarkLabel(lblEnd);
}
else
{
void EmitWrite(int rt, long offset)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
if (offset != 0)
{
context.EmitLdc_I8(offset);
context.Emit(OpCodes.Add);
}
context.EmitLdintzr(rt);
EmitWriteCall(context, op.Size);
}
EmitWrite(op.Rt, 0);
if (pair)
{
EmitWrite(op.Rt2, 1 << op.Size);
}
}
}