// TSrcs (64bit) == TDst (64bit); signed, unsigned.
public static void EmitBinarySatQSub(ILEmitterCtx context, bool signed)
{
if (((OpCodeSimdReg64)context.CurrOp).Size < 3)
{
throw new InvalidOperationException();
}
context.EmitLdarg(TranslatedSub.StateArgIdx);
SoftFallback.EmitCall(context, signed
? nameof(SoftFallback.BinarySignedSatQSub)
: nameof(SoftFallback.BinaryUnsignedSatQSub));
}
public static void Stp(ILEmitterCtx context)
{
OpCodeMemPair64 op = (OpCodeMemPair64)context.CurrOp;
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
if (op is IOpCodeSimd64)
{
context.EmitLdvec(op.Rt);
}
else
{
context.EmitLdintzr(op.Rt);
}
EmitWriteCall(context, op.Size);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);
context.Emit(OpCodes.Add);
if (op is IOpCodeSimd64)
{
context.EmitLdvec(op.Rt2);
}
else
{
context.EmitLdintzr(op.Rt2);
}
EmitWriteCall(context, op.Size);
EmitWBackIfNeeded(context);
}
public static void EmitSoftFloatCall(ILEmitterCtx context, string name)
{
IOpCodeSimd64 op = (IOpCodeSimd64)context.CurrOp;
int sizeF = op.Size & 1;
Type type = sizeF == 0
? typeof(SoftFloat32)
: typeof(SoftFloat64);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCall(type, name);
}
public static void EmitCall(ILEmitterCtx context, long imm)
{
if (context.Tier == TranslationTier.Tier0)
{
context.TranslateAhead(imm);
context.EmitLdc_I8(imm);
context.Emit(OpCodes.Ret);
return;
}
if (!context.TryOptEmitSubroutineCall())
{
context.TranslateAhead(imm);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitFieldLoad(typeof(CpuThreadState).GetField(nameof(CpuThreadState.CurrentTranslator),
BindingFlags.Instance |
BindingFlags.NonPublic));
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdc_I8(imm);
context.EmitPrivateCall(typeof(Translator), nameof(Translator.GetOrTranslateSubroutine));
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitCall(typeof(TranslatedSub), nameof(TranslatedSub.Execute));
}
EmitContinueOrReturnCheck(context);
}
public static void Fcvtn_V(ILEmitterCtx context)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
int sizeF = op.Size & 1;
int elems = 4 >> sizeF;
int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0;
if (part != 0)
{
context.EmitLdvec(op.Rd);
context.EmitStvectmp();
}
for (int index = 0; index < elems; index++)
{
EmitVectorExtractF(context, op.Rn, index, sizeF);
if (sizeF == 0)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCall(typeof(SoftFloat32_16), nameof(SoftFloat32_16.FPConvert));
context.Emit(OpCodes.Conv_U8);
EmitVectorInsertTmp(context, part + index, 1);
}
else /* if (sizeF == 1) */
{
context.Emit(OpCodes.Conv_R4);
EmitVectorInsertTmpF(context, part + index, 0);
}
}
context.EmitLdvectmp();
context.EmitStvec(op.Rd);
if (part == 0)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
public static void Mrs(ILEmitterCtx context)
{
OpCodeSystem64 op = (OpCodeSystem64)context.CurrOp;
context.EmitLdarg(TranslatedSub.StateArgIdx);
string propName;
switch (GetPackedId(op))
{
case 0b11_011_0000_0000_001: propName = nameof(CpuThreadState.CtrEl0); break;
case 0b11_011_0000_0000_111: propName = nameof(CpuThreadState.DczidEl0); break;
case 0b11_011_0100_0100_000: propName = nameof(CpuThreadState.Fpcr); break;
case 0b11_011_0100_0100_001: propName = nameof(CpuThreadState.Fpsr); break;
case 0b11_011_1101_0000_010: propName = nameof(CpuThreadState.TpidrEl0); break;
case 0b11_011_1101_0000_011: propName = nameof(CpuThreadState.Tpidr); break;
case 0b11_011_1110_0000_000: propName = nameof(CpuThreadState.CntfrqEl0); break;
case 0b11_011_1110_0000_001: propName = nameof(CpuThreadState.CntpctEl0); break;
default: throw new NotImplementedException($"Unknown MRS at {op.Position:x16}");
}
context.EmitCallPropGet(typeof(CpuThreadState), propName);
PropertyInfo propInfo = typeof(CpuThreadState).GetProperty(propName);
if (propInfo.PropertyType != typeof(long) &&
propInfo.PropertyType != typeof(ulong))
{
context.Emit(OpCodes.Conv_U8);
}
context.EmitStintzr(op.Rt);
}
// TSrc (16bit, 32bit, 64bit; signed, unsigned) > TDst (8bit, 16bit, 32bit; signed, unsigned).
public static void EmitSatQ(ILEmitterCtx context, int sizeDst, bool signedSrc, bool signedDst)
{
if ((uint)sizeDst > 2u)
{
throw new ArgumentOutOfRangeException(nameof(sizeDst));
}
context.EmitLdc_I4(sizeDst);
context.EmitLdarg(TranslatedSub.StateArgIdx);
if (signedSrc)
{
SoftFallback.EmitCall(context, signedDst
? nameof(SoftFallback.SignedSrcSignedDstSatQ)
: nameof(SoftFallback.SignedSrcUnsignedDstSatQ));
}
else
{
SoftFallback.EmitCall(context, signedDst
? nameof(SoftFallback.UnsignedSrcSignedDstSatQ)
: nameof(SoftFallback.UnsignedSrcUnsignedDstSatQ));
}
}
public static void Und(ILEmitterCtx context)
{
OpCode64 op = context.CurrOp;
context.EmitStoreState();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdc_I8(op.Position);
context.EmitLdc_I4(op.RawOpCode);
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.OnUndefined));
if (context.CurrBlock.Next != null)
{
context.EmitLoadState(context.CurrBlock.Next);
}
else
{
context.EmitLdc_I8(op.Position + 4);
context.Emit(OpCodes.Ret);
}
}
public static void Fcvtn_V(ILEmitterCtx context)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
int sizeF = op.Size & 1;
if (Optimizations.UseSse2 && sizeF == 1)
{
Type[] typesCvt = new Type[] { typeof(Vector128 <double>) };
string nameMov = op.RegisterSize == RegisterSize.Simd128
? nameof(Sse.MoveLowToHigh)
: nameof(Sse.MoveHighToLow);
context.EmitLdvec(op.Rd);
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorSingleZero));
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.MoveLowToHigh)));
EmitLdvecWithCastToDouble(context, op.Rn);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.ConvertToVector128Single), typesCvt));
context.Emit(OpCodes.Dup);
context.EmitCall(typeof(Sse).GetMethod(nameof(Sse.MoveLowToHigh)));
context.EmitCall(typeof(Sse).GetMethod(nameMov));
context.EmitStvec(op.Rd);
}
else
{
int elems = 4 >> sizeF;
int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0;
if (part != 0)
{
context.EmitLdvec(op.Rd);
context.EmitStvectmp();
}
for (int index = 0; index < elems; index++)
{
EmitVectorExtractF(context, op.Rn, index, sizeF);
if (sizeF == 0)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCall(typeof(SoftFloat32_16), nameof(SoftFloat32_16.FPConvert));
context.Emit(OpCodes.Conv_U8);
EmitVectorInsertTmp(context, part + index, 1);
}
else /* if (sizeF == 1) */
{
context.Emit(OpCodes.Conv_R4);
EmitVectorInsertTmpF(context, part + index, 0);
}
}
context.EmitLdvectmp();
context.EmitStvec(op.Rd);
if (part == 0)
{
EmitVectorZeroUpper(context, op.Rd);
}
}
}
public static void Fcvt_S(ILEmitterCtx context)
{
OpCodeSimd64 op = (OpCodeSimd64)context.CurrOp;
if (op.Size == 0 && op.Opc == 1) // Single -> Double.
{
if (Optimizations.UseSse2)
{
Type[] typesCvt = new Type[] { typeof(Vector128 <double>), typeof(Vector128 <float>) };
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorSingleZero));
context.EmitLdvec(op.Rn);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.ConvertScalarToVector128Double), typesCvt));
context.EmitStvec(op.Rd);
}
else
{
EmitVectorExtractF(context, op.Rn, 0, 0);
EmitFloatCast(context, 1);
EmitScalarSetF(context, op.Rd, 1);
}
}
else if (op.Size == 1 && op.Opc == 0) // Double -> Single.
{
if (Optimizations.UseSse2)
{
Type[] typesCvt = new Type[] { typeof(Vector128 <float>), typeof(Vector128 <double>) };
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorSingleZero));
context.EmitLdvec(op.Rn);
context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.ConvertScalarToVector128Single), typesCvt));
context.EmitStvec(op.Rd);
}
else
{
EmitVectorExtractF(context, op.Rn, 0, 1);
EmitFloatCast(context, 0);
EmitScalarSetF(context, op.Rd, 0);
}
}
else if (op.Size == 0 && op.Opc == 3) // Single -> Half.
{
EmitVectorExtractF(context, op.Rn, 0, 0);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCall(typeof(SoftFloat32_16), nameof(SoftFloat32_16.FPConvert));
context.Emit(OpCodes.Conv_U8);
EmitScalarSet(context, op.Rd, 1);
}
else if (op.Size == 3 && op.Opc == 0) // Half -> Single.
{
EmitVectorExtractZx(context, op.Rn, 0, 1);
context.Emit(OpCodes.Conv_U2);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCall(typeof(SoftFloat16_32), nameof(SoftFloat16_32.FPConvert));
EmitScalarSetF(context, op.Rd, 0);
}
else if (op.Size == 1 && op.Opc == 3) // Double -> Half.
{
throw new NotImplementedException("Double-precision to half-precision.");
}
else if (op.Size == 3 && op.Opc == 1) // Double -> Half.
{
throw new NotImplementedException("Half-precision to double-precision.");
}
else // Invalid encoding.
{
throw new InvalidOperationException($"type == {op.Size} && opc == {op.Opc}");
}
}
private static void EmitSimdMemSs(ILEmitterCtx context, bool isLoad)
{
OpCodeSimdMemSs64 op = (OpCodeSimdMemSs64)context.CurrOp;
int offset = 0;
void EmitMemAddress()
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(offset);
context.Emit(OpCodes.Add);
}
if (op.Replicate)
{
//Only loads uses the replicate mode.
if (!isLoad)
{
throw new InvalidOperationException();
}
int bytes = op.GetBitsCount() >> 3;
int elems = bytes >> op.Size;
for (int sElem = 0; sElem < op.SElems; sElem++)
{
int rt = (op.Rt + sElem) & 0x1f;
for (int index = 0; index < elems; index++)
{
EmitMemAddress();
EmitReadZxCall(context, op.Size);
EmitVectorInsert(context, rt, index, op.Size);
}
if (op.RegisterSize == RegisterSize.Simd64)
{
EmitVectorZeroUpper(context, rt);
}
offset += 1 << op.Size;
}
}
else
{
for (int sElem = 0; sElem < op.SElems; sElem++)
{
int rt = (op.Rt + sElem) & 0x1f;
if (isLoad)
{
EmitMemAddress();
EmitReadZxCall(context, op.Size);
EmitVectorInsert(context, rt, op.Index, op.Size);
}
else
{
EmitMemAddress();
EmitVectorExtractZx(context, rt, op.Index, op.Size);
EmitWriteCall(context, op.Size);
}
offset += 1 << op.Size;
}
}
if (op.WBack)
{
EmitSimdMemWBack(context, offset);
}
}
private static void EmitLoad(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
context.EmitLdint(op.Rn);
context.EmitSttmp();
if (exclusive)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.SetExclusiveAddress));
}
void WriteExclusiveValue(string propName)
{
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U8);
}
context.EmitSttmp2();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdtmp2();
context.EmitCallPrivatePropSet(typeof(CpuThreadState), propName);
context.EmitLdtmp2();
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
}
if (pair)
{
//Exclusive loads should be atomic. For pairwise loads, we need to
//read all the data at once. For a 32-bits pairwise load, we do a
//simple 64-bits load, for a 128-bits load, we need to call a special
//method to read 128-bits atomically.
if (op.Size == 2)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, 3);
context.Emit(OpCodes.Dup);
//Mask low half.
context.Emit(OpCodes.Conv_U4);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
//Shift high half.
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh));
}
context.EmitStintzr(op.Rt2);
}
else if (op.Size == 3)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicReadInt128));
context.Emit(OpCodes.Dup);
//Load low part of the vector.
context.EmitLdc_I4(0);
context.EmitLdc_I4(3);
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx));
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
//Load high part of the vector.
context.EmitLdc_I4(1);
context.EmitLdc_I4(3);
VectorHelper.EmitCall(context, nameof(VectorHelper.VectorExtractIntZx));
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueHigh));
}
context.EmitStintzr(op.Rt2);
}
else
{
throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
else
{
//8, 16, 32 or 64-bits (non-pairwise) load.
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, op.Size);
if (exclusive)
{
WriteExclusiveValue(nameof(CpuThreadState.ExclusiveValueLow));
}
context.EmitStintzr(op.Rt);
}
}
public static void Clrex(ILEmitterCtx context)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.ClearExclusiveAddress));
}
private static void EmitStore(ILEmitterCtx context, AccessType accType, bool pair)
{
OpCodeMemEx64 op = (OpCodeMemEx64)context.CurrOp;
bool ordered = (accType & AccessType.Ordered) != 0;
bool exclusive = (accType & AccessType.Exclusive) != 0;
if (ordered)
{
EmitBarrier(context);
}
if (exclusive)
{
ILLabel lblEx = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitLdint(op.Rn);
context.EmitPrivateCall(typeof(CpuThreadState), nameof(CpuThreadState.CheckExclusiveAddress));
context.Emit(OpCodes.Brtrue_S, lblEx);
//Address check failed, set error right away and do not store anything.
context.EmitLdc_I4(1);
context.EmitStintzr(op.Rs);
context.Emit(OpCodes.Br, lblEnd);
//Address check passsed.
context.MarkLabel(lblEx);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueLow));
void EmitCast()
{
//The input should be always int64.
switch (op.Size)
{
case 0: context.Emit(OpCodes.Conv_U1); break;
case 1: context.Emit(OpCodes.Conv_U2); break;
case 2: context.Emit(OpCodes.Conv_U4); break;
}
}
EmitCast();
if (pair)
{
context.EmitLdarg(TranslatedSub.StateArgIdx);
context.EmitCallPrivatePropGet(typeof(CpuThreadState), nameof(CpuThreadState.ExclusiveValueHigh));
EmitCast();
context.EmitLdintzr(op.Rt);
EmitCast();
context.EmitLdintzr(op.Rt2);
EmitCast();
switch (op.Size)
{
case 2: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchange2xInt32)); break;
case 3: context.EmitPrivateCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt128)); break;
default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
else
{
context.EmitLdintzr(op.Rt);
EmitCast();
switch (op.Size)
{
case 0: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeByte)); break;
case 1: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt16)); break;
case 2: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt32)); break;
case 3: context.EmitCall(typeof(MemoryManager), nameof(MemoryManager.AtomicCompareExchangeInt64)); break;
default: throw new InvalidOperationException($"Invalid store size of {1 << op.Size} bytes.");
}
}
//The value returned is a bool, true if the values compared
//were equal and the new value was written, false otherwise.
//We need to invert this result, as on ARM 1 indicates failure,
//and 0 success on those instructions.
context.EmitLdc_I4(1);
context.Emit(OpCodes.Xor);
context.Emit(OpCodes.Dup);
context.Emit(OpCodes.Conv_U8);
context.EmitStintzr(op.Rs);
//Only clear the exclusive monitor if the store was successful (Rs = false).
context.Emit(OpCodes.Brtrue_S, lblEnd);
Clrex(context);
context.MarkLabel(lblEnd);
}
else
{
void EmitWrite(int rt, long offset)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
if (offset != 0)
{
context.EmitLdc_I8(offset);
context.Emit(OpCodes.Add);
}
context.EmitLdintzr(rt);
EmitWriteCall(context, op.Size);
}
EmitWrite(op.Rt, 0);
if (pair)
{
EmitWrite(op.Rt2, 1 << op.Size);
}
}
}
private static void EmitLoadOrStore(ILEmitterCtx context, int size, AccessType accType)
{
OpCode32Mem op = (OpCode32Mem)context.CurrOp;
if (op.Index || op.WBack)
{
EmitLoadFromRegister(context, op.Rn);
context.EmitLdc_I4(op.Imm);
context.Emit(op.Add ? OpCodes.Add : OpCodes.Sub);
context.EmitSttmp();
}
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
if (op.Index)
{
context.EmitLdtmp();
}
else
{
EmitLoadFromRegister(context, op.Rn);
}
if ((accType & AccessType.Load) != 0)
{
if ((accType & AccessType.Signed) != 0)
{
EmitReadSx32Call(context, size);
}
else
{
EmitReadZxCall(context, size);
}
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Dup);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsr(32);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt);
context.Emit(OpCodes.Conv_U4);
EmitStoreToRegister(context, op.Rt | 1);
context.MarkLabel(lblEnd);
}
else
{
EmitStoreToRegister(context, op.Rt);
}
}
else
{
if (op.WBack)
{
context.EmitLdtmp();
EmitStoreToRegister(context, op.Rn);
}
EmitLoadFromRegister(context, op.Rt);
if (size == DWordSizeLog2)
{
context.Emit(OpCodes.Conv_U8);
context.EmitLdflg((int)PState.EBit);
ILLabel lblBigEndian = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brtrue_S, lblBigEndian);
//Little endian mode.
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.EmitLsl(32);
context.Emit(OpCodes.Or);
context.Emit(OpCodes.Br_S, lblEnd);
//Big endian mode.
context.MarkLabel(lblBigEndian);
context.EmitLsl(32);
EmitLoadFromRegister(context, op.Rt | 1);
context.Emit(OpCodes.Conv_U8);
context.Emit(OpCodes.Or);
context.MarkLabel(lblEnd);
}
EmitWriteCall(context, size);
}
}