private static void EmitBifBit(ArmEmitterContext context, bool notRm)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (Optimizations.UseSse2)
{
EmitVectorTernaryOpSimd32(context, (d, n, m) =>
{
Operand res = context.AddIntrinsic(Intrinsic.X86Pxor, n, d);
res = context.AddIntrinsic((notRm) ? Intrinsic.X86Pandn : Intrinsic.X86Pand, m, res);
return(context.AddIntrinsic(Intrinsic.X86Pxor, d, res));
});
}
else
{
EmitVectorTernaryOpZx32(context, (d, n, m) =>
{
if (notRm)
{
m = context.BitwiseNot(m);
}
return(context.BitwiseExclusiveOr(
context.BitwiseAnd(m,
context.BitwiseExclusiveOr(d, n)), d));
});
}
}
public static Operand GetLslC(ArmEmitterContext context, Operand m, bool setCarry, int shift)
{
Debug.Assert(m.Type == OperandType.I32);
if ((uint)shift > 32)
{
return(GetShiftByMoreThan32(context, setCarry));
}
else if (shift == 32)
{
if (setCarry)
{
SetCarryMLsb(context, m);
}
return(Const(0));
}
else
{
if (setCarry)
{
Operand cOut = context.ShiftRightUI(m, Const(32 - shift));
cOut = context.BitwiseAnd(cOut, Const(1));
SetFlag(context, PState.CFlag, cOut);
}
return(context.ShiftLeft(m, Const(shift)));
}
}
private static void EmitCmtstOp(ArmEmitterContext context, bool scalar)
{
OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp;
Operand res = context.VectorZero();
int elems = !scalar?op.GetBytesCount() >> op.Size : 1;
ulong szMask = ulong.MaxValue >> (64 - (8 << op.Size));
for (int index = 0; index < elems; index++)
{
Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size);
Operand me = EmitVectorExtractZx(context, op.Rm, index, op.Size);
Operand test = context.BitwiseAnd(ne, me);
Operand isTrue = context.ICompareNotEqual(test, Const(0L));
Operand mask = context.ConditionalSelect(isTrue, Const(szMask), Const(0L));
res = EmitVectorInsert(context, res, mask, index, op.Size);
}
context.Copy(GetVec(op.Rd), res);
}
private static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblFallbackPath)
{
Operand pte = Const(context.Memory.PageTable.ToInt64());
int bit = MemoryManager.PageBits;
do
{
Operand addrPart = context.ShiftRightUI(address, Const(bit));
bit += context.Memory.PtLevelBits;
if (bit < context.Memory.AddressSpaceBits)
{
addrPart = context.BitwiseAnd(addrPart, Const(addrPart.Type, context.Memory.PtLevelMask));
}
Operand pteOffset = context.ShiftLeft(addrPart, Const(3));
if (pteOffset.Type == OperandType.I32)
{
pteOffset = context.ZeroExtend32(OperandType.I64, pteOffset);
}
Operand pteAddress = context.Add(pte, pteOffset);
pte = context.Load(OperandType.I64, pteAddress);
}while (bit < context.Memory.AddressSpaceBits);
if (!context.Memory.HasWriteWatchSupport)
{
Operand hasFlagSet = context.BitwiseAnd(pte, Const((long)MemoryManager.PteFlagsMask));
context.BranchIfTrue(lblFallbackPath, hasFlagSet);
}
Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, MemoryManager.PageMask));
if (pageOffset.Type == OperandType.I32)
{
pageOffset = context.ZeroExtend32(OperandType.I64, pageOffset);
}
Operand physAddr = context.Add(pte, pageOffset);
return(physAddr);
}
private static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblSlowPath)
{
int ptLevelBits = context.Memory.AddressSpaceBits - 12; // 12 = Number of page bits.
int ptLevelSize = 1 << ptLevelBits;
int ptLevelMask = ptLevelSize - 1;
Operand pte = Ptc.State == PtcState.Disabled
? Const(context.Memory.PageTablePointer.ToInt64())
: Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex);
int bit = PageBits;
do
{
Operand addrPart = context.ShiftRightUI(address, Const(bit));
bit += ptLevelBits;
if (bit < context.Memory.AddressSpaceBits)
{
addrPart = context.BitwiseAnd(addrPart, Const(addrPart.Type, ptLevelMask));
}
Operand pteOffset = context.ShiftLeft(addrPart, Const(3));
if (pteOffset.Type == OperandType.I32)
{
pteOffset = context.ZeroExtend32(OperandType.I64, pteOffset);
}
Operand pteAddress = context.Add(pte, pteOffset);
pte = context.Load(OperandType.I64, pteAddress);
}while (bit < context.Memory.AddressSpaceBits);
context.BranchIfTrue(lblSlowPath, context.ICompareLess(pte, Const(0L)));
Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, PageMask));
if (pageOffset.Type == OperandType.I32)
{
pageOffset = context.ZeroExtend32(OperandType.I64, pageOffset);
}
return(context.Add(pte, pageOffset));
}
public static void Vtst(ArmEmitterContext context)
{
EmitVectorBinaryOpZx32(context, (op1, op2) =>
{
Operand isZero = context.ICompareEqual(context.BitwiseAnd(op1, op2), Const(0));
return(context.ConditionalSelect(isZero, Const(0), Const(-1)));
});
}
public static void Uhadd8(ArmEmitterContext context)
{
OpCode32AluReg op = (OpCode32AluReg)context.CurrOp;
Operand m = GetIntA32(context, op.Rm);
Operand n = GetIntA32(context, op.Rn);
Operand xor, res;
res = context.BitwiseAnd(m, n);
xor = context.BitwiseExclusiveOr(m, n);
xor = context.ShiftRightUI(xor, Const(1));
xor = context.BitwiseAnd(xor, Const(0x7F7F7F7Fu));
res = context.Add(res, xor);
SetIntA32(context, op.Rd, res);
}
public static void EmitBxWritePc(ArmEmitterContext context, Operand pc)
{
Operand mode = context.BitwiseAnd(pc, Const(1));
SetFlag(context, PState.TFlag, mode);
Operand lblArmMode = Label();
context.BranchIfTrue(lblArmMode, mode);
// Make this count as a call, the translator will ignore the low bit for the address.
context.Return(context.ZeroExtend32(OperandType.I64, context.BitwiseOr(pc, Const((int)InstEmitFlowHelper.CallFlag))));
context.MarkLabel(lblArmMode);
context.Return(context.ZeroExtend32(OperandType.I64, context.BitwiseOr(context.BitwiseAnd(pc, Const(~3)), Const((int)InstEmitFlowHelper.CallFlag))));
}
private static void EmitTb(ArmEmitterContext context, bool onNotZero)
{
OpCodeBImmTest op = (OpCodeBImmTest)context.CurrOp;
Operand value = context.BitwiseAnd(GetIntOrZR(context, op.Rt), Const(1L << op.Bit));
EmitBranch(context, value, onNotZero);
}
private static Operand EmitAddressCheck(ArmEmitterContext context, Operand address, int size)
{
ulong addressCheckMask = ~((1UL << context.Memory.AddressSpaceBits) - 1);
addressCheckMask |= (1u << size) - 1;
return(context.BitwiseAnd(address, Const(address.Type, (long)addressCheckMask)));
}
private static void SetCarryMShrOut(ArmEmitterContext context, Operand m, int shift)
{
Operand cOut = context.ShiftRightUI(m, Const(shift - 1));
cOut = context.BitwiseAnd(cOut, Const(1));
SetFlag(context, PState.CFlag, cOut);
}
private static void EmitBifBit(ArmEmitterContext context, bool notRm)
{
OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp;
if (Optimizations.UseSse2)
{
Operand d = GetVec(op.Rd);
Operand n = GetVec(op.Rn);
Operand m = GetVec(op.Rm);
Operand res = context.AddIntrinsic(Intrinsic.X86Pxor, n, d);
if (notRm)
{
res = context.AddIntrinsic(Intrinsic.X86Pandn, m, res);
}
else
{
res = context.AddIntrinsic(Intrinsic.X86Pand, m, res);
}
res = context.AddIntrinsic(Intrinsic.X86Pxor, d, res);
if (op.RegisterSize == RegisterSize.Simd64)
{
res = context.VectorZeroUpper64(res);
}
context.Copy(GetVec(op.Rd), res);
}
else
{
Operand res = context.VectorZero();
int elems = op.RegisterSize == RegisterSize.Simd128 ? 2 : 1;
for (int index = 0; index < elems; index++)
{
Operand d = EmitVectorExtractZx(context, op.Rd, index, 3);
Operand n = EmitVectorExtractZx(context, op.Rn, index, 3);
Operand m = EmitVectorExtractZx(context, op.Rm, index, 3);
if (notRm)
{
m = context.BitwiseNot(m);
}
Operand e = context.BitwiseExclusiveOr(d, n);
e = context.BitwiseAnd(e, m);
e = context.BitwiseExclusiveOr(e, d);
res = EmitVectorInsert(context, res, e, index, 3);
}
context.Copy(GetVec(op.Rd), res);
}
}
public static Operand EmitLoadExclusive(ArmEmitterContext context, Operand address, bool exclusive, int size)
{
if (exclusive)
{
Operand value;
if (size == 4)
{
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
context.MarkLabel(lblFastPath);
// Only 128-bit CAS is guaranteed to have a atomic load.
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false);
Operand zero = context.VectorZero();
value = context.CompareAndSwap(physAddr, zero, zero);
}
else
{
value = InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
}
Operand arg0 = context.LoadArgument(OperandType.I64, 0);
Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
context.Store(exAddrPtr, context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask())));
// Make sure the unused higher bits of the value are cleared.
if (size < 3)
{
context.Store(exValuePtr, Const(0UL));
}
if (size < 4)
{
context.Store(context.Add(exValuePtr, Const(exValuePtr.Type, 8L)), Const(0UL));
}
// Store the new exclusive value.
context.Store(exValuePtr, value);
return(value);
}
else
{
return(InstEmitMemoryHelper.EmitReadIntAligned(context, address, size));
}
}
public static void Vbsl(ArmEmitterContext context)
{
EmitVectorTernaryOpZx32(context, (op1, op2, op3) =>
{
return(context.BitwiseExclusiveOr(
context.BitwiseAnd(op1,
context.BitwiseExclusiveOr(op2, op3)), op3));
});
}
public static void Bfc(ArmEmitterContext context)
{
OpCode32AluBf op = (OpCode32AluBf)context.CurrOp;
Operand d = GetIntA32(context, op.Rd);
Operand res = context.BitwiseAnd(d, Const(~op.DestMask));
SetIntA32(context, op.Rd, res);
}
private static Operand EmitReverseBits64Op(ArmEmitterContext context, Operand op)
{
Debug.Assert(op.Type == OperandType.I64);
Operand val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op, Const(0xaaaaaaaaaaaaaaaaul)), Const(1)),
context.ShiftLeft(context.BitwiseAnd(op, Const(0x5555555555555555ul)), Const(1)));
val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xccccccccccccccccul)), Const(2)),
context.ShiftLeft(context.BitwiseAnd(val, Const(0x3333333333333333ul)), Const(2)));
val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xf0f0f0f0f0f0f0f0ul)), Const(4)),
context.ShiftLeft(context.BitwiseAnd(val, Const(0x0f0f0f0f0f0f0f0ful)), Const(4)));
val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xff00ff00ff00ff00ul)), Const(8)),
context.ShiftLeft(context.BitwiseAnd(val, Const(0x00ff00ff00ff00fful)), Const(8)));
val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xffff0000ffff0000ul)), Const(16)),
context.ShiftLeft(context.BitwiseAnd(val, Const(0x0000ffff0000fffful)), Const(16)));
return(context.BitwiseOr(context.ShiftRightUI(val, Const(32)), context.ShiftLeft(val, Const(32))));
}
public static void Tst(ArmEmitterContext context)
{
Operand n = GetAluN(context);
Operand m = GetAluM(context);
Operand res = context.BitwiseAnd(n, m);
EmitNZFlagsCheck(context, res);
}
public static void Bfi(ArmEmitterContext context)
{
OpCode32AluBf op = (OpCode32AluBf)context.CurrOp;
Operand n = GetIntA32(context, op.Rn);
Operand d = GetIntA32(context, op.Rd);
Operand part = context.BitwiseAnd(n, Const(op.SourceMask));
if (op.Lsb != 0)
{
part = context.ShiftLeft(part, Const(op.Lsb));
}
Operand res = context.BitwiseAnd(d, Const(~op.DestMask));
res = context.BitwiseOr(res, context.BitwiseAnd(part, Const(op.DestMask)));
SetIntA32(context, op.Rd, res);
}
private static Operand GetBfmN(ArmEmitterContext context)
{
OpCodeBfm op = (OpCodeBfm)context.CurrOp;
Operand res = GetIntOrZR(context, op.Rn);
long mask = op.WMask & op.TMask;
return(context.BitwiseAnd(context.RotateRight(res, Const(op.Shift)), Const(res.Type, mask)));
}
private static void EmitSri(ArmEmitterContext context, bool scalar)
{
OpCodeSimdShImm op = (OpCodeSimdShImm)context.CurrOp;
int shift = GetImmShr(op);
int eSize = 8 << op.Size;
ulong mask = (ulong.MaxValue << (eSize - shift)) & (ulong.MaxValue >> (64 - eSize));
if (Optimizations.UseSse2 && op.Size > 0)
{
Operand d = GetVec(op.Rd);
Operand n = GetVec(op.Rn);
Intrinsic srlInst = X86PsrlInstruction[op.Size];
Operand nShifted = context.AddIntrinsic(srlInst, n, Const(shift));
Operand dMask = X86GetAllElements(context, (long)mask * _masks_SliSri[op.Size]);
Operand dMasked = context.AddIntrinsic(Intrinsic.X86Pand, d, dMask);
Operand res = context.AddIntrinsic(Intrinsic.X86Por, nShifted, dMasked);
if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
{
res = context.VectorZeroUpper64(res);
}
context.Copy(d, res);
}
else
{
Operand res = context.VectorZero();
int elems = !scalar?op.GetBytesCount() >> op.Size : 1;
for (int index = 0; index < elems; index++)
{
Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size);
Operand neShifted = shift != 64 ? context.ShiftRightUI(ne, Const(shift)) : Const(0UL);
Operand de = EmitVectorExtractZx(context, op.Rd, index, op.Size);
Operand deMasked = context.BitwiseAnd(de, Const(mask));
Operand e = context.BitwiseOr(neShifted, deMasked);
res = EmitVectorInsert(context, res, e, index, op.Size);
}
context.Copy(GetVec(op.Rd), res);
}
}
public static void Vbic_I(ArmEmitterContext context)
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(Intrinsic.X86Pandn, m, n));
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.BitwiseAnd(op1, context.BitwiseNot(op2)));
}
}
public static void EmitBxWritePc(ArmEmitterContext context, Operand pc, int sourceRegister = 0)
{
bool isReturn = sourceRegister == RegisterAlias.Aarch32Lr || IsA32Return(context);
Operand mode = context.BitwiseAnd(pc, Const(1));
SetFlag(context, PState.TFlag, mode);
Operand addr = context.ConditionalSelect(mode, context.BitwiseOr(pc, Const((int)InstEmitFlowHelper.CallFlag)), context.BitwiseAnd(pc, Const(~3)));
InstEmitFlowHelper.EmitVirtualJump(context, addr, isReturn);
}
public static void EmitSbcsCCheck(ArmEmitterContext context, Operand n, Operand m)
{
// C = (Rn == Rm && CIn) || Rn > Rm
Operand cIn = GetFlag(PState.CFlag);
Operand cOut = context.BitwiseAnd(context.ICompareEqual(n, m), cIn);
cOut = context.BitwiseOr(cOut, context.ICompareGreaterUI(n, m));
SetFlag(context, PState.CFlag, cOut);
}
public static void Vand_I(ArmEmitterContext context)
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Pand, Intrinsic.X86Pand);
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.BitwiseAnd(op1, op2));
}
}
public static void EmitAdcsCCheck(ArmEmitterContext context, Operand n, Operand d)
{
// C = (Rd == Rn && CIn) || Rd < Rn
Operand cIn = GetFlag(PState.CFlag);
Operand cOut = context.BitwiseAnd(context.ICompareEqual(d, n), cIn);
cOut = context.BitwiseOr(cOut, context.ICompareLessUI(d, n));
SetFlag(context, PState.CFlag, cOut);
}
public static void EmitSubsVCheck(ArmEmitterContext context, Operand n, Operand m, Operand d)
{
// V = (Rd ^ Rn) & (Rn ^ Rm) < 0
Operand vOut = context.BitwiseExclusiveOr(d, n);
vOut = context.BitwiseAnd(vOut, context.BitwiseExclusiveOr(n, m));
vOut = context.ICompareLess(vOut, Const(vOut.Type, 0));
SetFlag(context, PState.VFlag, vOut);
}
public static void Ands(ArmEmitterContext context)
{
Operand n = GetAluN(context);
Operand m = GetAluM(context);
Operand d = context.BitwiseAnd(n, m);
EmitNZFlagsCheck(context, d);
EmitCVFlagsClear(context);
SetAluDOrZR(context, d);
}
public static void Movt(ArmEmitterContext context)
{
OpCode32AluImm16 op = (OpCode32AluImm16)context.CurrOp;
Operand d = GetIntA32(context, op.Rd);
Operand imm = Const(op.Immediate << 16); // Immeditate value as top halfword.
Operand res = context.BitwiseAnd(d, Const(0x0000ffff));
res = context.BitwiseOr(res, imm);
EmitAluStore(context, res);
}
private static void EmitAluWritePc(ArmEmitterContext context, Operand value)
{
context.StoreToContext();
if (IsThumb(context.CurrOp))
{
context.Return(context.ZeroExtend32(OperandType.I64, context.BitwiseAnd(value, Const(~1))));
}
else
{
EmitBxWritePc(context, value);
}
}
public static void Pkh(ArmEmitterContext context)
{
OpCode32AluRsImm op = (OpCode32AluRsImm)context.CurrOp;
Operand n = GetAluN(context);
Operand m = GetAluM(context);
Operand res;
bool tbform = op.ShiftType == ShiftType.Asr;
if (tbform)
{
res = context.BitwiseOr(context.BitwiseAnd(n, Const(0xFFFF0000)), context.BitwiseAnd(m, Const(0xFFFF)));
}
else
{
res = context.BitwiseOr(context.BitwiseAnd(m, Const(0xFFFF0000)), context.BitwiseAnd(n, Const(0xFFFF)));
}
EmitAluStore(context, res);
}