public static void Vmov_GD(ArmEmitterContext context)
{
OpCode32SimdMovGpDouble op = (OpCode32SimdMovGpDouble)context.CurrOp;
Operand vec = GetVecA32(op.Vm >> 1);
if (op.Op == 1)
{
// To general purpose.
Operand value = context.VectorExtract(OperandType.I64, vec, op.Vm & 1);
SetIntA32(context, op.Rt, context.ConvertI64ToI32(value));
SetIntA32(context, op.Rt2, context.ConvertI64ToI32(context.ShiftRightUI(value, Const(32))));
}
else
{
// From general purpose.
Operand lowValue = GetIntA32(context, op.Rt);
Operand highValue = GetIntA32(context, op.Rt2);
Operand value = context.BitwiseOr(
context.ZeroExtend32(OperandType.I64, lowValue),
context.ShiftLeft(context.ZeroExtend32(OperandType.I64, highValue), Const(32)));
context.Copy(vec, context.VectorInsert(vec, value, op.Vm & 1));
}
}
public static void Mrrc(ArmEmitterContext context)
{
OpCode32System op = (OpCode32System)context.CurrOp;
if (op.Coproc != 15)
{
throw new NotImplementedException($"Unknown MRC Coprocessor ID 0x{op.Coproc:X16} at 0x{op.Address:X16}.");
}
var opc = op.MrrcOp;
Delegate dlg;
switch (op.CRm)
{
case 14: // Timer.
switch (opc)
{
case 0:
dlg = new _U64(NativeInterface.GetCntpctEl0); break;
default:
throw new NotImplementedException($"Unknown MRRC Opc1 0x{opc:X16} at 0x{op.Address:X16}.");
}
break;
default:
throw new NotImplementedException($"Unknown MRRC 0x{op.RawOpCode:X8} at 0x{op.Address:X16}.");
}
Operand result = context.Call(dlg);
SetIntA32(context, op.Rt, context.ConvertI64ToI32(result));
SetIntA32(context, op.CRn, context.ConvertI64ToI32(context.ShiftRightUI(result, Const(32))));
}
public static void Vrshr(ArmEmitterContext context)
{
OpCode32SimdShImm op = (OpCode32SimdShImm)context.CurrOp;
int shift = GetImmShr(op);
long roundConst = 1L << (shift - 1);
if (op.U)
{
if (op.Size < 2)
{
EmitVectorUnaryOpZx32(context, (op1) =>
{
op1 = context.Add(op1, Const(op1.Type, roundConst));
return(context.ShiftRightUI(op1, Const(shift)));
});
}
else if (op.Size == 2)
{
EmitVectorUnaryOpZx32(context, (op1) =>
{
op1 = context.ZeroExtend32(OperandType.I64, op1);
op1 = context.Add(op1, Const(op1.Type, roundConst));
return(context.ConvertI64ToI32(context.ShiftRightUI(op1, Const(shift))));
});
}
else /* if (op.Size == 3) */
{
EmitVectorUnaryOpZx32(context, (op1) => EmitShrImm64(context, op1, signed: false, roundConst, shift));
}
}
else
{
if (op.Size < 2)
{
EmitVectorUnaryOpSx32(context, (op1) =>
{
op1 = context.Add(op1, Const(op1.Type, roundConst));
return(context.ShiftRightSI(op1, Const(shift)));
});
}
else if (op.Size == 2)
{
EmitVectorUnaryOpSx32(context, (op1) =>
{
op1 = context.SignExtend32(OperandType.I64, op1);
op1 = context.Add(op1, Const(op1.Type, roundConst));
return(context.ConvertI64ToI32(context.ShiftRightSI(op1, Const(shift))));
});
}
else /* if (op.Size == 3) */
{
EmitVectorUnaryOpZx32(context, (op1) => EmitShrImm64(context, op1, signed: true, roundConst, shift));
}
}
}
public static Operand EmitCrc32(ArmEmitterContext context, Operand crc, Operand value, int size, bool castagnoli)
{
Debug.Assert(crc.Type.IsInteger() && value.Type.IsInteger());
Debug.Assert(size >= 0 && size < 4);
Debug.Assert((size < 3) || (value.Type == OperandType.I64));
if (castagnoli && Optimizations.UseSse42)
{
// The CRC32 instruction does not have an immediate variant, so ensure both inputs are in registers.
value = (value.Kind == OperandKind.Constant) ? context.Copy(value) : value;
crc = (crc.Kind == OperandKind.Constant) ? context.Copy(crc) : crc;
Intrinsic op = size switch
{
0 => Intrinsic.X86Crc32_8,
1 => Intrinsic.X86Crc32_16,
_ => Intrinsic.X86Crc32,
};
return((size == 3) ? context.ConvertI64ToI32(context.AddIntrinsicLong(op, crc, value)) : context.AddIntrinsicInt(op, crc, value));
}
else if (Optimizations.UsePclmulqdq)
{
return(size switch
{
3 => EmitCrc32Optimized64(context, crc, value, castagnoli),
_ => EmitCrc32Optimized(context, crc, value, castagnoli, size),
});
public static void Smlaw_(ArmEmitterContext context)
{
OpCode32AluMla op = (OpCode32AluMla)context.CurrOp;
Operand n = GetIntA32(context, op.Rn);
Operand m = GetIntA32(context, op.Rm);
Operand a = GetIntA32(context, op.Ra);
if (op.MHigh)
{
m = context.SignExtend16(OperandType.I64, context.ShiftRightUI(m, Const(16)));
}
else
{
m = context.SignExtend16(OperandType.I64, m);
}
Operand res = context.Multiply(context.SignExtend32(OperandType.I64, n), m);
Operand toAdd = context.ShiftLeft(context.SignExtend32(OperandType.I64, a), Const(16));
res = context.Add(res, toAdd);
res = context.ShiftRightSI(res, Const(16));
Operand q = context.ICompareNotEqual(res, context.SignExtend32(OperandType.I64, res));
res = context.ConvertI64ToI32(res);
UpdateQFlag(context, q);
EmitGenericAluStoreA32(context, op.Rd, false, res);
}
private static void EmitSetNzcv(ArmEmitterContext context)
{
OpCodeSystem op = (OpCodeSystem)context.CurrOp;
Operand t = GetIntOrZR(context, op.Rt);
t = context.ConvertI64ToI32(t);
Operand v = context.ShiftRightUI(t, Const((int)PState.VFlag));
v = context.BitwiseAnd(v, Const(1));
Operand c = context.ShiftRightUI(t, Const((int)PState.CFlag));
c = context.BitwiseAnd(c, Const(1));
Operand z = context.ShiftRightUI(t, Const((int)PState.ZFlag));
z = context.BitwiseAnd(z, Const(1));
Operand n = context.ShiftRightUI(t, Const((int)PState.NFlag));
n = context.BitwiseAnd(n, Const(1));
SetFlag(context, PState.VFlag, v);
SetFlag(context, PState.CFlag, c);
SetFlag(context, PState.ZFlag, z);
SetFlag(context, PState.NFlag, n);
}
private static void EmitSmmul(ArmEmitterContext context, MullFlags flags)
{
OpCode32AluMla op = (OpCode32AluMla)context.CurrOp;
Operand n = context.SignExtend32(OperandType.I64, GetIntA32(context, op.Rn));
Operand m = context.SignExtend32(OperandType.I64, GetIntA32(context, op.Rm));
Operand res = context.Multiply(n, m);
if (flags.HasFlag(MullFlags.Add) && op.Ra != 0xf)
{
res = context.Add(context.ShiftLeft(context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Ra)), Const(32)), res);
}
else if (flags.HasFlag(MullFlags.Subtract))
{
res = context.Subtract(context.ShiftLeft(context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Ra)), Const(32)), res);
}
if (op.R)
{
res = context.Add(res, Const(0x80000000L));
}
Operand hi = context.ConvertI64ToI32(context.ShiftRightSI(res, Const(32)));
EmitGenericAluStoreA32(context, op.Rd, false, hi);
}
public static Operand EmitReverseBytes16_32Op(ArmEmitterContext context, Operand op)
{
Debug.Assert(op.Type == OperandType.I32);
Operand val = EmitReverseBytes16_64Op(context, context.ZeroExtend32(OperandType.I64, op));
return(context.ConvertI64ToI32(val));
}
private static void EmitLoadEx(ArmEmitterContext context, AccessType accType, bool pair)
{
OpCodeMemEx op = (OpCodeMemEx)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
Operand address = context.Copy(GetIntOrSP(context, op.Rn));
if (pair)
{
// Exclusive loads should be atomic. For pairwise loads, we need to
// read all the data at once. For a 32-bits pairwise load, we do a
// simple 64-bits load, for a 128-bits load, we need to call a special
// method to read 128-bits atomically.
if (op.Size == 2)
{
Operand value = EmitLoadExclusive(context, address, exclusive, 3);
Operand valueLow = context.ConvertI64ToI32(value);
valueLow = context.ZeroExtend32(OperandType.I64, valueLow);
Operand valueHigh = context.ShiftRightUI(value, Const(32));
SetIntOrZR(context, op.Rt, valueLow);
SetIntOrZR(context, op.Rt2, valueHigh);
}
else if (op.Size == 3)
{
Operand value = EmitLoadExclusive(context, address, exclusive, 4);
Operand valueLow = context.VectorExtract(OperandType.I64, value, 0);
Operand valueHigh = context.VectorExtract(OperandType.I64, value, 1);
SetIntOrZR(context, op.Rt, valueLow);
SetIntOrZR(context, op.Rt2, valueHigh);
}
else
{
throw new InvalidOperationException($"Invalid load size of {1 << op.Size} bytes.");
}
}
else
{
// 8, 16, 32 or 64-bits (non-pairwise) load.
Operand value = EmitLoadExclusive(context, address, exclusive, op.Size);
SetIntOrZR(context, op.Rt, value);
}
if (ordered)
{
// Memory operations after this load may be only executed after the load completes.
EmitBarrier(context);
}
}
public static void Umaal(ArmEmitterContext context)
{
OpCode32AluUmull op = (OpCode32AluUmull)context.CurrOp;
Operand n = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rn));
Operand m = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rm));
Operand dHi = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdHi));
Operand dLo = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdLo));
Operand res = context.Multiply(n, m);
res = context.Add(res, dHi);
res = context.Add(res, dLo);
Operand hi = context.ConvertI64ToI32(context.ShiftRightUI(res, Const(32)));
Operand lo = context.ConvertI64ToI32(res);
EmitGenericAluStoreA32(context, op.RdHi, false, hi);
EmitGenericAluStoreA32(context, op.RdLo, false, lo);
}
public static Operand GetIntOrSP(ArmEmitterContext context, int regIndex)
{
Operand value = Register(regIndex, RegisterType.Integer, OperandType.I64);
if (context.CurrOp.RegisterSize == RegisterSize.Int32)
{
value = context.ConvertI64ToI32(value);
}
return(value);
}
public static void Umull(ArmEmitterContext context)
{
OpCode32AluUmull op = (OpCode32AluUmull)context.CurrOp;
Operand n = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rn));
Operand m = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rm));
Operand res = context.Multiply(n, m);
Operand hi = context.ConvertI64ToI32(context.ShiftRightUI(res, Const(32)));
Operand lo = context.ConvertI64ToI32(res);
if (op.SetFlags)
{
EmitNZFlagsCheck(context, res);
}
EmitGenericAluStoreA32(context, op.RdHi, op.SetFlags, hi);
EmitGenericAluStoreA32(context, op.RdLo, op.SetFlags, lo);
}
private static Operand GetAluMShift(ArmEmitterContext context)
{
IOpCodeAluRs op = (IOpCodeAluRs)context.CurrOp;
Operand m = GetIntOrZR(context, op.Rm);
if (op.RegisterSize == RegisterSize.Int64)
{
m = context.ConvertI64ToI32(m);
}
return(context.BitwiseAnd(m, Const(context.CurrOp.GetBitsCount() - 1)));
}
public static Operand EmitStoreExclusive(
ArmEmitterContext context,
Operand address,
Operand value,
bool exclusive,
int size)
{
if (size < 3)
{
value = context.ConvertI64ToI32(value);
}
Delegate fallbackMethodDlg = null;
if (exclusive)
{
switch (size)
{
case 0: fallbackMethodDlg = new _S32_U64_U8(NativeInterface.WriteByteExclusive); break;
case 1: fallbackMethodDlg = new _S32_U64_U16(NativeInterface.WriteUInt16Exclusive); break;
case 2: fallbackMethodDlg = new _S32_U64_U32(NativeInterface.WriteUInt32Exclusive); break;
case 3: fallbackMethodDlg = new _S32_U64_U64(NativeInterface.WriteUInt64Exclusive); break;
case 4: fallbackMethodDlg = new _S32_U64_V128(NativeInterface.WriteVector128Exclusive); break;
}
return(context.Call(fallbackMethodDlg, address, value));
}
else
{
switch (size)
{
case 0: fallbackMethodDlg = new _Void_U64_U8(NativeInterface.WriteByte); break;
case 1: fallbackMethodDlg = new _Void_U64_U16(NativeInterface.WriteUInt16); break;
case 2: fallbackMethodDlg = new _Void_U64_U32(NativeInterface.WriteUInt32); break;
case 3: fallbackMethodDlg = new _Void_U64_U64(NativeInterface.WriteUInt64); break;
case 4: fallbackMethodDlg = new _Void_U64_V128(NativeInterface.WriteVector128); break;
}
context.Call(fallbackMethodDlg, address, value);
return(null);
}
}
public static Operand EmitStoreExclusive(
ArmEmitterContext context,
Operand address,
Operand value,
bool exclusive,
int size)
{
if (size < 3)
{
value = context.ConvertI64ToI32(value);
}
MethodInfo info = null;
if (exclusive)
{
switch (size)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByteExclusive)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16Exclusive)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32Exclusive)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64Exclusive)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128Exclusive)); break;
}
return(context.Call(info, address, value));
}
else
{
switch (size)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); break;
}
context.Call(info, address, value);
return(null);
}
}
public static void Mrrc(ArmEmitterContext context)
{
OpCode32System op = (OpCode32System)context.CurrOp;
if (op.Coproc != 15)
{
InstEmit.Und(context);
return;
}
int opc = op.MrrcOp;
MethodInfo info;
switch (op.CRm)
{
case 14: // Timer.
switch (opc)
{
case 0:
info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0)); break;
default:
throw new NotImplementedException($"Unknown MRRC Opc1 0x{opc:X16} at 0x{op.Address:X16}.");
}
break;
default:
throw new NotImplementedException($"Unknown MRRC 0x{op.RawOpCode:X8} at 0x{op.Address:X16}.");
}
Operand result = context.Call(info);
SetIntA32(context, op.Rt, context.ConvertI64ToI32(result));
SetIntA32(context, op.CRn, context.ConvertI64ToI32(context.ShiftRightUI(result, Const(32))));
}
public static void Smlal__(ArmEmitterContext context)
{
OpCode32AluUmull op = (OpCode32AluUmull)context.CurrOp;
Operand n = GetIntA32(context, op.Rn);
Operand m = GetIntA32(context, op.Rm);
if (op.NHigh)
{
n = context.SignExtend16(OperandType.I64, context.ShiftRightUI(n, Const(16)));
}
else
{
n = context.SignExtend16(OperandType.I64, n);
}
if (op.MHigh)
{
m = context.SignExtend16(OperandType.I64, context.ShiftRightUI(m, Const(16)));
}
else
{
m = context.SignExtend16(OperandType.I64, m);
}
Operand res = context.Multiply(n, m);
Operand toAdd = context.ShiftLeft(context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdHi)), Const(32));
toAdd = context.BitwiseOr(toAdd, context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdLo)));
res = context.Add(res, toAdd);
Operand hi = context.ConvertI64ToI32(context.ShiftRightUI(res, Const(32)));
Operand lo = context.ConvertI64ToI32(res);
EmitGenericAluStoreA32(context, op.RdHi, false, hi);
EmitGenericAluStoreA32(context, op.RdLo, false, lo);
}
public static void EmitMlal(ArmEmitterContext context, bool signed)
{
OpCode32AluUmull op = (OpCode32AluUmull)context.CurrOp;
Operand n = GetIntA32(context, op.Rn);
Operand m = GetIntA32(context, op.Rm);
if (signed)
{
n = context.SignExtend32(OperandType.I64, n);
m = context.SignExtend32(OperandType.I64, m);
}
else
{
n = context.ZeroExtend32(OperandType.I64, n);
m = context.ZeroExtend32(OperandType.I64, m);
}
Operand res = context.Multiply(n, m);
Operand toAdd = context.ShiftLeft(context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdHi)), Const(32));
toAdd = context.BitwiseOr(toAdd, context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.RdLo)));
res = context.Add(res, toAdd);
Operand hi = context.ConvertI64ToI32(context.ShiftRightUI(res, Const(32)));
Operand lo = context.ConvertI64ToI32(res);
if (op.SetFlags)
{
EmitNZFlagsCheck(context, res);
}
EmitGenericAluStoreA32(context, op.RdHi, op.SetFlags, hi);
EmitGenericAluStoreA32(context, op.RdLo, op.SetFlags, lo);
}
public static void SetIntA32(ArmEmitterContext context, int regIndex, Operand value)
{
if (regIndex == RegisterAlias.Aarch32Pc)
{
context.StoreToContext();
EmitBxWritePc(context, value);
}
else
{
if (value.Type == OperandType.I64)
{
value = context.ConvertI64ToI32(value);
}
Operand reg = Register(GetRegisterAlias(context.Mode, regIndex), RegisterType.Integer, OperandType.I32);
context.Copy(reg, value);
}
}
public static void Ushl_V(ArmEmitterContext context)
{
OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp;
Operand res = context.VectorZero();
int elems = op.GetBytesCount() >> op.Size;
for (int index = 0; index < elems; index++)
{
Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size);
Operand me = EmitVectorExtractSx(context, op.Rm, index << op.Size, 0);
Operand e = EmitUnsignedShlRegOp(context, ne, context.ConvertI64ToI32(me), op.Size);
res = EmitVectorInsert(context, res, e, index, op.Size);
}
context.Copy(GetVec(op.Rd), res);
}
private static void EmitSshlOrUshl(ArmEmitterContext context, bool signed, bool scalar)
{
OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp;
Operand res = context.VectorZero();
int elems = !scalar?op.GetBytesCount() >> op.Size : 1;
for (int index = 0; index < elems; index++)
{
Operand ne = EmitVectorExtract(context, op.Rn, index, op.Size, signed);
Operand me = EmitVectorExtractSx(context, op.Rm, index << op.Size, 0);
Operand e = EmitShlRegOp(context, ne, context.ConvertI64ToI32(me), op.Size, signed);
res = EmitVectorInsert(context, res, e, index, op.Size);
}
context.Copy(GetVec(op.Rd), res);
}
private static void EmitWriteInt(ArmEmitterContext context, Operand address, int rt, int size)
{
Operand isUnalignedAddr = EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
Operand lblSlowPath = Label();
Operand lblEnd = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
context.MarkLabel(lblSlowPath);
EmitWriteIntFallback(context, address, rt, size);
context.Branch(lblEnd);
context.MarkLabel(lblFastPath);
Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath);
Operand value = GetInt(context, rt);
if (size < 3 && value.Type == OperandType.I64)
{
value = context.ConvertI64ToI32(value);
}
switch (size)
{
case 0: context.Store8(physAddr, value); break;
case 1: context.Store16(physAddr, value); break;
case 2: context.Store(physAddr, value); break;
case 3: context.Store(physAddr, value); break;
}
context.MarkLabel(lblEnd);
}
private static Operand EmitShlRegOp(ArmEmitterContext context, Operand op, Operand shiftLsB, int size, bool unsigned)
{
if (shiftLsB.Type == OperandType.I64)
{
shiftLsB = context.ConvertI64ToI32(shiftLsB);
}
shiftLsB = context.SignExtend8(OperandType.I32, shiftLsB);
Debug.Assert((uint)size < 4u);
Operand negShiftLsB = context.Negate(shiftLsB);
Operand isPositive = context.ICompareGreaterOrEqual(shiftLsB, Const(0));
Operand shl = context.ShiftLeft(op, shiftLsB);
Operand shr = unsigned ? context.ShiftRightUI(op, negShiftLsB) : context.ShiftRightSI(op, negShiftLsB);
Operand res = context.ConditionalSelect(isPositive, shl, shr);
if (unsigned)
{
Operand isOutOfRange = context.BitwiseOr(
context.ICompareGreaterOrEqual(shiftLsB, Const(8 << size)),
context.ICompareGreaterOrEqual(negShiftLsB, Const(8 << size)));
return(context.ConditionalSelect(isOutOfRange, Const(op.Type, 0), res));
}
else
{
Operand isOutOfRange0 = context.ICompareGreaterOrEqual(shiftLsB, Const(8 << size));
Operand isOutOfRangeN = context.ICompareGreaterOrEqual(negShiftLsB, Const(8 << size));
// Also zero if shift is too negative, but value was positive.
isOutOfRange0 = context.BitwiseOr(isOutOfRange0, context.BitwiseAnd(isOutOfRangeN, context.ICompareGreaterOrEqual(op, Const(op.Type, 0))));
Operand min = (op.Type == OperandType.I64) ? Const(-1L) : Const(-1);
return(context.ConditionalSelect(isOutOfRange0, Const(op.Type, 0), context.ConditionalSelect(isOutOfRangeN, min, res)));
}
}
public static void Rbit_V(ArmEmitterContext context)
{
OpCodeSimd op = (OpCodeSimd)context.CurrOp;
Operand res = context.VectorZero();
int elems = op.RegisterSize == RegisterSize.Simd128 ? 16 : 8;
for (int index = 0; index < elems; index++)
{
Operand ne = EmitVectorExtractZx(context, op.Rn, index, 0);
ne = context.ConvertI64ToI32(ne);
Operand de = context.Call(new _U32_U32(SoftFallback.ReverseBits8), ne);
de = context.ZeroExtend32(OperandType.I64, de);
res = EmitVectorInsert(context, res, de, index, 0);
}
context.Copy(GetVec(op.Rd), res);
}
public static void Smulw_(ArmEmitterContext context)
{
OpCode32AluMla op = (OpCode32AluMla)context.CurrOp;
Operand n = GetIntA32(context, op.Rn);
Operand m = GetIntA32(context, op.Rm);
if (op.MHigh)
{
m = context.SignExtend16(OperandType.I64, context.ShiftRightUI(m, Const(16)));
}
else
{
m = context.SignExtend16(OperandType.I64, m);
}
Operand res = context.Multiply(context.SignExtend32(OperandType.I64, n), m);
res = context.ShiftRightUI(res, Const(16));
res = context.ConvertI64ToI32(res);
EmitGenericAluStoreA32(context, op.Rd, false, res);
}
private static void EmitWriteIntFallback(ArmEmitterContext context, Operand address, int rt, int size)
{
MethodInfo info = null;
switch (size)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); break;
}
Operand value = GetInt(context, rt);
if (size < 3 && value.Type == OperandType.I64)
{
value = context.ConvertI64ToI32(value);
}
context.Call(info, address, value);
}
private static void EmitWriteIntFallback(ArmEmitterContext context, Operand address, int rt, int size)
{
Delegate fallbackMethodDlg = null;
switch (size)
{
case 0: fallbackMethodDlg = new _Void_U64_U8(NativeInterface.WriteByte); break;
case 1: fallbackMethodDlg = new _Void_U64_U16(NativeInterface.WriteUInt16); break;
case 2: fallbackMethodDlg = new _Void_U64_U32(NativeInterface.WriteUInt32); break;
case 3: fallbackMethodDlg = new _Void_U64_U64(NativeInterface.WriteUInt64); break;
}
Operand value = GetInt(context, rt);
if (size < 3 && value.Type == OperandType.I64)
{
value = context.ConvertI64ToI32(value);
}
context.Call(fallbackMethodDlg, address, value);
}
public static void EmitStoreExclusive(
ArmEmitterContext context,
Operand address,
Operand value,
bool exclusive,
int size,
int rs,
bool a32)
{
if (size < 3)
{
value = context.ConvertI64ToI32(value);
}
if (exclusive)
{
void SetRs(Operand value)
{
if (a32)
{
SetIntA32(context, rs, value);
}
else
{
SetIntOrZR(context, rs, value);
}
}
Operand arg0 = context.LoadArgument(OperandType.I64, 0);
Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
Operand exAddr = context.Load(address.Type, exAddrPtr);
// STEP 1: Check if we have exclusive access to this memory region. If not, fail and skip store.
Operand maskedAddress = context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask()));
Operand exFailed = context.ICompareNotEqual(exAddr, maskedAddress);
Operand lblExit = Label();
SetRs(exFailed);
context.BranchIfTrue(lblExit, exFailed);
// STEP 2: We have exclusive access, make sure that the address is valid.
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
// STEP 3: We have exclusive access and the address is valid, attempt the store using CAS.
context.MarkLabel(lblFastPath);
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true);
Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
Operand exValue = size switch
{
0 => context.Load8(exValuePtr),
1 => context.Load16(exValuePtr),
2 => context.Load(OperandType.I32, exValuePtr),
3 => context.Load(OperandType.I64, exValuePtr),
_ => context.Load(OperandType.V128, exValuePtr)
};
Operand currValue = size switch
{
0 => context.CompareAndSwap8(physAddr, exValue, value),
1 => context.CompareAndSwap16(physAddr, exValue, value),
_ => context.CompareAndSwap(physAddr, exValue, value)
};
// STEP 4: Check if we succeeded by comparing expected and in-memory values.
Operand storeFailed;
if (size == 4)
{
Operand currValueLow = context.VectorExtract(OperandType.I64, currValue, 0);
Operand currValueHigh = context.VectorExtract(OperandType.I64, currValue, 1);
Operand exValueLow = context.VectorExtract(OperandType.I64, exValue, 0);
Operand exValueHigh = context.VectorExtract(OperandType.I64, exValue, 1);
storeFailed = context.BitwiseOr(
context.ICompareNotEqual(currValueLow, exValueLow),
context.ICompareNotEqual(currValueHigh, exValueHigh));
}
else
{
storeFailed = context.ICompareNotEqual(currValue, exValue);
}
SetRs(storeFailed);
context.MarkLabel(lblExit);
}
else
{
InstEmitMemoryHelper.EmitWriteIntAligned(context, address, value, size);
}
}
private static void EmitSse41ConvertInt32(ArmEmitterContext context, FPRoundingMode roundMode, bool signed)
{
// A port of the similar round function in InstEmitSimdCvt.
OpCode32SimdCvtFI op = (OpCode32SimdCvtFI)context.CurrOp;
bool doubleSize = (op.Size & 1) != 0;
int shift = doubleSize ? 1 : 2;
Operand n = GetVecA32(op.Vm >> shift);
n = EmitSwapScalar(context, n, op.Vm, doubleSize);
if (!doubleSize)
{
Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, n, n, Const((int)CmpCondition.OrderedQ));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n);
nRes = context.AddIntrinsic(Intrinsic.X86Roundss, nRes, Const(X86GetRoundControl(roundMode)));
Operand zero = context.VectorZero();
Operand nCmp;
Operand nIntOrLong2 = default;
if (!signed)
{
nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp);
}
int fpMaxVal = 0x4F000000; // 2.14748365E9f (2147483648)
Operand fpMaxValMask = X86GetScalar(context, fpMaxVal);
Operand nIntOrLong = context.AddIntrinsicInt(Intrinsic.X86Cvtss2si, nRes);
if (!signed)
{
nRes = context.AddIntrinsic(Intrinsic.X86Subss, nRes, fpMaxValMask);
nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp);
nIntOrLong2 = context.AddIntrinsicInt(Intrinsic.X86Cvtss2si, nRes);
}
nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan));
Operand nInt = context.AddIntrinsicInt(Intrinsic.X86Cvtsi2si, nRes);
Operand dRes;
if (signed)
{
dRes = context.BitwiseExclusiveOr(nIntOrLong, nInt);
}
else
{
dRes = context.BitwiseExclusiveOr(nIntOrLong2, nInt);
dRes = context.Add(dRes, nIntOrLong);
}
InsertScalar(context, op.Vd, dRes);
}
else
{
Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, n, n, Const((int)CmpCondition.OrderedQ));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n);
nRes = context.AddIntrinsic(Intrinsic.X86Roundsd, nRes, Const(X86GetRoundControl(roundMode)));
Operand zero = context.VectorZero();
Operand nCmp;
Operand nIntOrLong2 = default;
if (!signed)
{
nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp);
}
long fpMaxVal = 0x41E0000000000000L; // 2147483648.0000000d (2147483648)
Operand fpMaxValMask = X86GetScalar(context, fpMaxVal);
Operand nIntOrLong = context.AddIntrinsicInt(Intrinsic.X86Cvtsd2si, nRes);
if (!signed)
{
nRes = context.AddIntrinsic(Intrinsic.X86Subsd, nRes, fpMaxValMask);
nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp);
nIntOrLong2 = context.AddIntrinsicInt(Intrinsic.X86Cvtsd2si, nRes);
}
nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan));
Operand nLong = context.AddIntrinsicLong(Intrinsic.X86Cvtsi2si, nRes);
nLong = context.ConvertI64ToI32(nLong);
Operand dRes;
if (signed)
{
dRes = context.BitwiseExclusiveOr(nIntOrLong, nLong);
}
else
{
dRes = context.BitwiseExclusiveOr(nIntOrLong2, nLong);
dRes = context.Add(dRes, nIntOrLong);
}
InsertScalar(context, op.Vd, dRes);
}
}
private static void EmitExLoadOrStore(ArmEmitterContext context, int size, AccessType accType)
{
IOpCode32MemEx op = (IOpCode32MemEx)context.CurrOp;
Operand address = context.Copy(GetIntA32(context, op.Rn));
var exclusive = (accType & AccessType.Exclusive) != 0;
var ordered = (accType & AccessType.Ordered) != 0;
if ((accType & AccessType.Load) != 0)
{
if (ordered)
{
EmitBarrier(context);
}
if (size == DWordSizeLog2)
{
// Keep loads atomic - make the call to get the whole region and then decompose it into parts
// for the registers.
Operand value = EmitLoadExclusive(context, address, exclusive, size);
Operand valueLow = context.ConvertI64ToI32(value);
valueLow = context.ZeroExtend32(OperandType.I64, valueLow);
Operand valueHigh = context.ShiftRightUI(value, Const(32));
Operand lblBigEndian = Label();
Operand lblEnd = Label();
context.BranchIfTrue(lblBigEndian, GetFlag(PState.EFlag));
SetIntA32(context, op.Rt, valueLow);
SetIntA32(context, op.Rt | 1, valueHigh);
context.Branch(lblEnd);
context.MarkLabel(lblBigEndian);
SetIntA32(context, op.Rt | 1, valueLow);
SetIntA32(context, op.Rt, valueHigh);
context.MarkLabel(lblEnd);
}
else
{
SetIntA32(context, op.Rt, EmitLoadExclusive(context, address, exclusive, size));
}
}
else
{
if (size == DWordSizeLog2)
{
// Split the result into 2 words (based on endianness)
Operand lo = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rt));
Operand hi = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rt | 1));
Operand lblBigEndian = Label();
Operand lblEnd = Label();
context.BranchIfTrue(lblBigEndian, GetFlag(PState.EFlag));
Operand leResult = context.BitwiseOr(lo, context.ShiftLeft(hi, Const(32)));
EmitStoreExclusive(context, address, leResult, exclusive, size, op.Rd, a32: true);
context.Branch(lblEnd);
context.MarkLabel(lblBigEndian);
Operand beResult = context.BitwiseOr(hi, context.ShiftLeft(lo, Const(32)));
EmitStoreExclusive(context, address, beResult, exclusive, size, op.Rd, a32: true);
context.MarkLabel(lblEnd);
}
else
{
Operand value = context.ZeroExtend32(OperandType.I64, GetIntA32(context, op.Rt));
EmitStoreExclusive(context, address, value, exclusive, size, op.Rd, a32: true);
}
if (ordered)
{
EmitBarrier(context);
}
}
}