private static void EmitVectorAddv(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
EmitVectorZeroLower(Context, Op.Rd);
EmitVectorZeroUpper(Context, Op.Rd);
Context.EmitLdvec(Op.Rd);
Context.EmitLdc_I4(0);
Context.EmitLdc_I4(Op.Size);
EmitVectorExtractZx(Context, Op.Rn, 0);
for (int Index = 1; Index < (Bytes >> Op.Size); Index++)
{
EmitVectorExtractZx(Context, Op.Rn, Index);
Context.Emit(OpCodes.Add);
}
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.InsertVec));
Context.EmitStvec(Op.Rd);
}
private static void EmitVectorZip(AILEmitterCtx Context, int Part)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
int Elems = Bytes >> Op.Size;
int Half = Elems >> 1;
for (int Index = 0; Index < Elems; Index++)
{
int Elem = Part * Half + (Index >> 1);
EmitVectorExtractZx(Context, (Index & 1) == 0 ? Op.Rn : Op.Rm, Elem, Op.Size);
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void Fcvtl_V(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
int SizeF = Op.Size & 1;
int Elems = 4 >> SizeF;
int Part = Op.RegisterSize == ARegisterSize.SIMD128 ? Elems : 0;
for (int Index = 0; Index < Elems; Index++)
{
if (SizeF == 0)
{
EmitVectorExtractZx(Context, Op.Rn, Part + Index, 1);
Context.Emit(OpCodes.Conv_U2);
Context.EmitLdarg(ATranslatedSub.StateArgIdx);
Context.EmitCall(typeof(ASoftFloat16_32), nameof(ASoftFloat16_32.FPConvert));
}
else /* if (SizeF == 1) */
{
EmitVectorExtractF(Context, Op.Rn, Part + Index, 0);
Context.Emit(OpCodes.Conv_R8);
}
EmitVectorInsertTmpF(Context, Index, SizeF);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
}
private static void EmitVectorZip(AILEmitterCtx Context, int Part)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int Words = Op.GetBitsCount() >> 4;
int Pairs = Words >> Op.Size;
int Base = Part != 0 ? Pairs : 0;
for (int Index = 0; Index < Pairs; Index++)
{
int Idx = Index << 1;
EmitVectorExtractZx(Context, Op.Rn, Base + Index, Op.Size);
EmitVectorExtractZx(Context, Op.Rm, Base + Index, Op.Size);
EmitVectorInsertTmp(Context, Idx + 1, Op.Size);
EmitVectorInsertTmp(Context, Idx, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
private static void EmitRev_V(AILEmitterCtx Context, int ContainerSize)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
if (Op.Size >= ContainerSize)
{
throw new InvalidOperationException();
}
int Bytes = Op.GetBitsCount() >> 3;
int Elems = Bytes >> Op.Size;
int ContainerMask = (1 << (ContainerSize - Op.Size)) - 1;
for (int Index = 0; Index < Elems; Index++)
{
int RevIndex = Index ^ ContainerMask;
EmitVectorExtractZx(Context, Op.Rn, RevIndex, Op.Size);
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void Faddp_V(AILEmitterCtx Context)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int SizeF = Op.Size & 1;
int Bytes = Op.GetBitsCount() >> 3;
int Elems = Bytes >> SizeF + 2;
int Half = Elems >> 1;
for (int Index = 0; Index < Elems; Index++)
{
int Elem = (Index & (Half - 1)) << 1;
EmitVectorExtractF(Context, Index < Half ? Op.Rn : Op.Rm, Elem + 0, SizeF);
EmitVectorExtractF(Context, Index < Half ? Op.Rn : Op.Rm, Elem + 1, SizeF);
Context.Emit(OpCodes.Add);
EmitVectorInsertTmpF(Context, Index, SizeF);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void LdrLit(AILEmitterCtx Context)
{
IAOpCodeLit Op = (IAOpCodeLit)Context.CurrOp;
if (Op.Prefetch)
{
return;
}
Context.EmitLdarg(ATranslatedSub.MemoryArgIdx);
Context.EmitLdc_I8(Op.Imm);
if (Op.Signed)
{
EmitReadSx64Call(Context, Op.Size);
}
else
{
EmitReadZxCall(Context, Op.Size);
}
if (Op is IAOpCodeSimd)
{
Context.EmitStvec(Op.Rt);
}
else
{
Context.EmitStint(Op.Rt);
}
}
public static void EmitSaturatingUnaryOpSx(AILEmitterCtx Context, Action Emit, SaturatingFlags Flags)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
bool Scalar = (Flags & SaturatingFlags.Scalar) != 0;
int Bytes = Op.GetBitsCount() >> 3;
int Elems = !Scalar ? Bytes >> Op.Size : 1;
if (Scalar)
{
EmitVectorZeroLowerTmp(Context);
}
for (int Index = 0; Index < Elems; Index++)
{
EmitVectorExtractSx(Context, Op.Rn, Index, Op.Size);
Emit();
EmitUnarySignedSatQAbsOrNeg(Context, Op.Size);
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if ((Op.RegisterSize == ARegisterSize.SIMD64) || Scalar)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitLdr(AILEmitterCtx Context, bool Signed)
{
AOpCodeMem Op = (AOpCodeMem)Context.CurrOp;
Context.EmitLdarg(ATranslatedSub.MemoryArgIdx);
EmitLoadAddress(Context);
if (Signed && Op.Extend64)
{
EmitReadSx64Call(Context, Op.Size);
}
else if (Signed)
{
EmitReadSx32Call(Context, Op.Size);
}
else
{
EmitReadZxCall(Context, Op.Size);
}
if (Op is IAOpCodeSimd)
{
Context.EmitStvec(Op.Rt);
}
else
{
Context.EmitStintzr(Op.Rt);
}
EmitWBackIfNeeded(Context);
}
public static void Xtn_V(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
int Elems = 8 >> Op.Size;
int Part = Op.RegisterSize == ARegisterSize.SIMD128 ? Elems : 0;
if (Part != 0)
{
Context.EmitLdvec(Op.Rd);
Context.EmitStvectmp();
}
for (int Index = 0; Index < Elems; Index++)
{
EmitVectorExtractZx(Context, Op.Rn, Index, Op.Size + 1);
EmitVectorInsertTmp(Context, Part + Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Part == 0)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void Ext_V(AILEmitterCtx Context)
{
AOpCodeSimdExt Op = (AOpCodeSimdExt)Context.CurrOp;
Context.EmitLdvec(Op.Rd);
Context.EmitStvectmp();
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
int Position = Op.Imm4;
for (int Index = 0; Index < Bytes; Index++)
{
int Reg = Op.Imm4 + Index < Bytes ? Op.Rn : Op.Rm;
if (Position == Bytes)
{
Position = 0;
}
EmitVectorExtractZx(Context, Reg, Position++, 0);
EmitVectorInsertTmp(Context, Index, 0);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
private static void EmitVectorImmBinaryOp(AILEmitterCtx Context, Action Emit, long Imm, bool Signed)
{
AOpCodeSimdShImm Op = (AOpCodeSimdShImm)Context.CurrOp;
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
for (int Index = 0; Index < (Bytes >> Op.Size); Index++)
{
Context.EmitLdvec(Op.Rd);
Context.EmitLdc_I4(Index);
Context.EmitLdc_I4(Op.Size);
EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
Context.EmitLdc_I8(Imm);
Emit();
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.InsertVec));
Context.EmitStvec(Op.Rd);
}
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitVectorWidenRnRmOp(AILEmitterCtx Context, Action Emit, bool Ternary, bool Signed)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
Context.EmitLdvec(Op.Rd);
Context.EmitStvectmp();
int Elems = 8 >> Op.Size;
int Part = Op.RegisterSize == ARegisterSize.SIMD128 ? Elems : 0;
for (int Index = 0; Index < Elems; Index++)
{
if (Ternary)
{
EmitVectorExtract(Context, Op.Rd, Index, Op.Size + 1, Signed);
}
EmitVectorExtract(Context, Op.Rn, Part + Index, Op.Size, Signed);
EmitVectorExtract(Context, Op.Rm, Part + Index, Op.Size, Signed);
Emit();
EmitVectorInsertTmp(Context, Index, Op.Size + 1);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
}
public static void Tbl_V(AILEmitterCtx Context)
{
AOpCodeSimdTbl Op = (AOpCodeSimdTbl)Context.CurrOp;
Context.EmitLdvec(Op.Rm);
for (int Index = 0; Index < Op.Size; Index++)
{
Context.EmitLdvec((Op.Rn + Index) & 0x1f);
}
switch (Op.Size)
{
case 1: AVectorHelper.EmitCall(Context,
nameof(AVectorHelper.Tbl1_V64),
nameof(AVectorHelper.Tbl1_V128)); break;
case 2: AVectorHelper.EmitCall(Context,
nameof(AVectorHelper.Tbl2_V64),
nameof(AVectorHelper.Tbl2_V128)); break;
case 3: AVectorHelper.EmitCall(Context,
nameof(AVectorHelper.Tbl3_V64),
nameof(AVectorHelper.Tbl3_V128)); break;
case 4: AVectorHelper.EmitCall(Context,
nameof(AVectorHelper.Tbl4_V64),
nameof(AVectorHelper.Tbl4_V128)); break;
default: throw new InvalidOperationException();
}
Context.EmitStvec(Op.Rd);
}
private static void EmitAddLongPairwise(AILEmitterCtx Context, bool Signed, bool Accumulate)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
int Words = Op.GetBitsCount() >> 4;
int Pairs = Words >> Op.Size;
for (int Index = 0; Index < Pairs; Index++)
{
int Idx = Index << 1;
EmitVectorExtract(Context, Op.Rn, Idx, Op.Size, Signed);
EmitVectorExtract(Context, Op.Rn, Idx + 1, Op.Size, Signed);
Context.Emit(OpCodes.Add);
if (Accumulate)
{
EmitVectorExtract(Context, Op.Rd, Index, Op.Size + 1, Signed);
Context.Emit(OpCodes.Add);
}
EmitVectorInsertTmp(Context, Index, Op.Size + 1);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
private static void EmitVectorPairwiseOp(AILEmitterCtx Context, Action Emit, bool Signed)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
int Elems = Bytes >> Op.Size;
int Half = Elems >> 1;
for (int Index = 0; Index < Elems; Index++)
{
int Elem = (Index & (Half - 1)) << 1;
EmitVectorExtract(Context, Index < Half ? Op.Rn : Op.Rm, Elem + 0, Op.Size, Signed);
EmitVectorExtract(Context, Index < Half ? Op.Rn : Op.Rm, Elem + 1, Op.Size, Signed);
Emit();
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitSse2CallF(AILEmitterCtx Context, string Name)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
int SizeF = Op.Size & 1;
void Ldvec(int Reg)
{
Context.EmitLdvec(Reg);
if (SizeF == 1)
{
AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToDouble));
}
}
Ldvec(Op.Rn);
Type BaseType = SizeF == 0
? typeof(Vector128 <float>)
: typeof(Vector128 <double>);
Type[] Types;
if (Op is AOpCodeSimdReg BinOp)
{
Ldvec(BinOp.Rm);
Types = new Type[] { BaseType, BaseType };
}
else
{
Types = new Type[] { BaseType };
}
MethodInfo MthdInfo;
if (SizeF == 0)
{
MthdInfo = typeof(Sse).GetMethod(Name, Types);
}
else /* if (SizeF == 1) */
{
MthdInfo = typeof(Sse2).GetMethod(Name, Types);
}
Context.EmitCall(MthdInfo);
if (SizeF == 1)
{
AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorDoubleToSingle));
}
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitVectorOpByElem(AILEmitterCtx Context, Action Emit, int Elem, bool Ternary, bool Signed)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int Bytes = Context.CurrOp.GetBitsCount() >> 3;
for (int Index = 0; Index < (Bytes >> Op.Size); Index++)
{
if (Ternary)
{
EmitVectorExtract(Context, Op.Rd, Index, Op.Size, Signed);
}
EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
EmitVectorExtract(Context, Op.Rm, Elem, Op.Size, Signed);
Emit();
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitVectorPairwiseOpF(AILEmitterCtx Context, Action Emit)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int SizeF = Op.Size & 1;
int Words = Op.GetBitsCount() >> 4;
int Pairs = Words >> SizeF + 2;
for (int Index = 0; Index < Pairs; Index++)
{
int Idx = Index << 1;
EmitVectorExtractF(Context, Op.Rn, Idx, SizeF);
EmitVectorExtractF(Context, Op.Rn, Idx + 1, SizeF);
Emit();
EmitVectorExtractF(Context, Op.Rm, Idx, SizeF);
EmitVectorExtractF(Context, Op.Rm, Idx + 1, SizeF);
Emit();
EmitVectorInsertTmpF(Context, Pairs + Index, SizeF);
EmitVectorInsertTmpF(Context, Index, SizeF);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitVectorOpByElemF(AILEmitterCtx Context, Action Emit, int Elem, bool Ternary)
{
AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
int SizeF = Op.Size & 1;
int Bytes = Op.GetBitsCount() >> 3;
int Elems = Bytes >> SizeF + 2;
for (int Index = 0; Index < Elems; Index++)
{
if (Ternary)
{
EmitVectorExtractF(Context, Op.Rd, Index, SizeF);
}
EmitVectorExtractF(Context, Op.Rn, Index, SizeF);
EmitVectorExtractF(Context, Op.Rm, Elem, SizeF);
Emit();
EmitVectorInsertTmpF(Context, Index, SizeF);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if (Op.RegisterSize == ARegisterSize.SIMD64)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
public static void EmitVectorZero32_128(AILEmitterCtx Context, int Reg)
{
Context.EmitLdvec(Reg);
AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorZero32_128));
Context.EmitStvec(Reg);
}
private static void EmitShrImmOp(AILEmitterCtx Context, ShrImmFlags Flags)
{
AOpCodeSimdShImm Op = (AOpCodeSimdShImm)Context.CurrOp;
bool Scalar = (Flags & ShrImmFlags.Scalar) != 0;
bool Signed = (Flags & ShrImmFlags.Signed) != 0;
bool Round = (Flags & ShrImmFlags.Round) != 0;
bool Accumulate = (Flags & ShrImmFlags.Accumulate) != 0;
int Shift = GetImmShr(Op);
long RoundConst = 1L << (Shift - 1);
int Bytes = Op.GetBitsCount() >> 3;
int Elems = !Scalar ? Bytes >> Op.Size : 1;
for (int Index = 0; Index < Elems; Index++)
{
EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
if (Op.Size <= 2)
{
if (Round)
{
Context.EmitLdc_I8(RoundConst);
Context.Emit(OpCodes.Add);
}
Context.EmitLdc_I4(Shift);
Context.Emit(Signed ? OpCodes.Shr : OpCodes.Shr_Un);
}
else /* if (Op.Size == 3) */
{
EmitShrImm_64(Context, Signed, Round ? RoundConst : 0L, Shift);
}
if (Accumulate)
{
EmitVectorExtract(Context, Op.Rd, Index, Op.Size, Signed);
Context.Emit(OpCodes.Add);
}
EmitVectorInsertTmp(Context, Index, Op.Size);
}
Context.EmitLdvectmp();
Context.EmitStvec(Op.Rd);
if ((Op.RegisterSize == ARegisterSize.SIMD64) || Scalar)
{
EmitVectorZeroUpper(Context, Op.Rd);
}
}
private static void EmitVectorInsert(AILEmitterCtx Context, int Reg, int Index, int Size)
{
Context.EmitLdvec(Reg);
Context.EmitLdc_I4(Index);
Context.EmitLdc_I4(Size);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.VectorInsertInt));
Context.EmitStvec(Reg);
}
public static void Aesmc_V(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
Context.EmitLdvec(Op.Rn);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.MixColumns));
Context.EmitStvec(Op.Rd);
}
public static void Addp_S(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
Context.EmitLdvec(Op.Rn);
Context.EmitLdc_I4(Op.Size);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.Addp_S));
Context.EmitStvec(Op.Rd);
}
public static void Sha256su0_V(AILEmitterCtx Context)
{
AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
Context.EmitLdvec(Op.Rd);
Context.EmitLdvec(Op.Rn);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.SchedulePart1));
Context.EmitStvec(Op.Rd);
}
public static void Dup_S(AILEmitterCtx Context)
{
AOpCodeSimdIns Op = (AOpCodeSimdIns)Context.CurrOp;
Context.EmitLdvec(Op.Rn);
Context.EmitLdc_I4(Op.DstIndex);
Context.EmitLdc_I4(Op.Size);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.Dup_S));
Context.EmitStvec(Op.Rd);
}
public static void Fmov_S(AILEmitterCtx Context)
{
AOpCodeSimdFmov Op = (AOpCodeSimdFmov)Context.CurrOp;
Context.EmitLdc_I8(Op.Imm);
Context.EmitLdc_I4(0);
Context.EmitLdc_I4(Op.Size + 2);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.Fmov_S));
Context.EmitStvec(Op.Rd);
}
public static void EmitVectorInsert(AILEmitterCtx Context, int Reg, int Index, int Size)
{
ThrowIfInvalid(Index, Size);
Context.EmitLdvec(Reg);
Context.EmitLdc_I4(Index);
Context.EmitLdc_I4(Size);
AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertInt));
Context.EmitStvec(Reg);
}
public static void Fmov_Itof1(AILEmitterCtx Context)
{
AOpCodeSimdCvt Op = (AOpCodeSimdCvt)Context.CurrOp;
Context.EmitLdintzr(Op.Rn);
Context.EmitLdc_I4(1);
Context.EmitLdc_I4(3);
ASoftFallback.EmitCall(Context, nameof(ASoftFallback.Fmov_S));
Context.EmitStvec(Op.Rd);
}