Example #1
0
        public static float DotMultiplyIntrinsicWAvxWSpanPtr(ref Memory <float> vector1, ref Memory <float> vector2)
        {
            var span1   = vector1.Span;
            var span2   = vector2.Span;
            var cnt     = Math.Min(span1.Length, span2.Length);
            var v3      = Vector256.CreateScalarUnsafe(0f);
            var vectLen = Vector256 <float> .Count;
            var vectCnt = cnt / vectLen;
            var total   = 0f;

#if TEST
            var file = Path.GetTempFileName();
            using var writer = new StreamWriter(file);
            Console.WriteLine($"Intrinsic with AvxWPtr Mult. results will be written into {file}");
#endif
            int i;
            unsafe
            {
                var ptr1 = (float *)Unsafe.AsPointer(ref span1[0]);
                var ptr2 = (float *)Unsafe.AsPointer(ref span2[0]);
                for (i = 0; i < vectCnt; i++)
                {
                    var v1 = Avx.LoadVector256(ptr1);
                    var v2 = Avx.LoadVector256(ptr2);
                    var t  = Avx.Multiply(v1, v2);
                    v3    = Avx.Add(v3, t);
                    ptr1 += vectLen;
                    ptr2 += vectLen;
#if TEST
                    writer.WriteLine($"{v1.ToString()}\t{v2.ToString()}\t{v3.ToString()}");
#endif
                }

                for (i = 0; i < vectLen; i++)
                {
                    total += v3.GetElement(i);
                }

                i = vectCnt * vectLen;
                if (cnt % vectLen > 0)
                {
                    ptr1 = (float *)Unsafe.AsPointer(ref span1[i]);
                    ptr2 = (float *)Unsafe.AsPointer(ref span2[i]);
                    for (; i < cnt; i++)
                    {
                        total += *ptr1++ **ptr2++;
                    }
                }
            }

            if (vector1.Length != vector2.Length)
            {
                var h = vector1.Length > vector2.Length ? span1 : span2;
                for (var j = cnt; j < h.Length; j++)
                {
                    total += h[j];
                }
            }

            return(total);
        }
        public unsafe void Vector256Mandel()
        {
            int floatL3Size = TOTALBYTES / sizeof(float);

            resolutionX = (int)MathF.Floor(MathF.Sqrt(floatL3Size * ratioy_x));
            if (resolutionX % 8 != 0)
            {
                resolutionX -= resolutionX % 8;
            }
            resolutionY = (int)MathF.Floor(resolutionX * ratioy_x);
            if (resolutionY % 8 != 0)
            {
                resolutionY -= resolutionY % 8;
            }
            STEP_X         = (RIGHT_X - LEFT_X) / resolutionX;
            STEP_Y         = STEP_X; // ratioy_x * STEP_X; Bug from reddit comment
            numberOfPoints = resolutionX * resolutionY;
            results2       = new float[numberOfPoints];

            xPoints = new float[resolutionX];
            yPoints = new float[resolutionY];
            for (int i = 0; i < resolutionX; i++)
            {
                xPoints.Span[i] = LEFT_X + i * STEP_X;
            }
            for (int i = 0; i < resolutionY; i++)
            {
                yPoints.Span[i] = TOP_Y - i * STEP_Y;
            }

            int countX = 0, countY = 0;
            int maxInter = 256;
            int inter;
            ReadOnlySpan <float> ySpan = yPoints.Span;// MemoryMarshal.Cast<float, Vector256<float>>(yPoints.Span);
            ReadOnlySpan <Vector256 <float> > xSpan    = MemoryMarshal.Cast <float, Vector256 <float> >(xPoints.Span);
            Span <Vector256 <float> >         res      = MemoryMarshal.Cast <float, Vector256 <float> >(results2.Span);
            Span <Vector256 <float> >         testSpan = MemoryMarshal.Cast <float, Vector256 <float> >(testValue2.Span);
            int resVectorNumber = 0;

            Vector256 <float> xVec, yVec;
            var oneVec  = Vector256.Create(1.0f);
            var fourVec = Vector256.Create(4.0f);

            while (countY < ySpan.Length)
            {
                var currYVec = Vector256.Create(ySpan[countY]);
                while (countX < xSpan.Length)
                {
                    Vector256 <float> currXVec = xSpan[countX];
                    var xSquVec  = Vector256.Create(0.0f);
                    var ySquVec  = Vector256.Create(0.0f);
                    var zSquVec  = Vector256.Create(0.0f);
                    var interVec = Vector256.Create(0.0f);
                    Vector256 <float> sumVector = oneVec;
                    inter = 0;
                    bool goOn = true;
                    while (goOn)
                    {
                        xVec    = Avx.Add(Avx.Subtract(xSquVec, ySquVec), currXVec);
                        yVec    = Avx.Add(Avx.Subtract(Avx.Subtract(zSquVec, ySquVec), xSquVec), currYVec);
                        xSquVec = Avx.Multiply(xVec, xVec);
                        ySquVec = Avx.Multiply(yVec, yVec);
                        zSquVec = Avx.Multiply(Avx.Add(xVec, yVec), Avx.Add(xVec, yVec));
                        Vector256 <float> test = Avx.Compare(Avx.Add(xSquVec, ySquVec), fourVec, FloatComparisonMode.OrderedLessThanOrEqualNonSignaling); // <= 4.0?
                        sumVector = Avx.BlendVariable(Vector256 <float> .Zero, sumVector, test);                                                          // selects from second if true, from first otherwise
                        goOn      = (Avx.MoveMask(test) > 0) & (inter < maxInter);                                                                        //any of the values still alive, and inter still below cutoff value?
                        if (goOn)
                        {
                            interVec = Avx.Add(interVec, sumVector);
                        }
                        inter = goOn ? inter + 1 : inter;
                    }
                    testSpan[resVectorNumber] = Avx.Add(xSquVec, ySquVec);
                    res[resVectorNumber]      = interVec;
                    resVectorNumber++;
                    countX++;
                }
                countX = 0;
                countY++;
            }
        }
Example #3
0
 // Element-wise multiplication.
 public static IEnumerable <Vector256 <double> > Mul(
     this IEnumerable <Vector256 <double> > @this,
     Vector256 <double> other)
 => @this.Select(v => Avx.Multiply(v, other));
        unsafe void IConvolver.SharpenLine(byte *cstart, byte *ystart, byte *bstart, byte *ostart, int ox, int ow, float amt, float thresh, bool gamma)
        {
            float *ip = (float *)cstart + (uint)ox * channels, yp = (float *)ystart + (uint)ox, bp = (float *)bstart, op = (float *)ostart;
            float *ipe = ip + (uint)ow * channels;

            bool threshold = thresh > 0f;

            if (Avx.IsSupported && ip <= ipe - VectorAvx.Count)
            {
                var vthresh = Vector256.Create(threshold ? thresh : -1f);
                var vmsk    = Vector256.Create(0x7fffffff).AsSingle();
                var vamt    = Vector256.Create(amt);
                var vmin    = VectorAvx.Zero;

                ipe -= VectorAvx.Count;
                do
                {
                    var vd = Avx.Subtract(Avx.LoadVector256(yp), Avx.LoadVector256(bp));
                    yp += VectorAvx.Count;
                    bp += VectorAvx.Count;

                    if (threshold)
                    {
                        var sm = HWIntrinsics.AvxCompareGreaterThan(Avx.And(vd, vmsk), vthresh);
                        vd = Avx.And(vd, sm);
                    }
                    vd = Avx.Multiply(vd, vamt);

                    var v0 = Avx.LoadVector256(ip);
                    ip += VectorAvx.Count;

                    if (gamma)
                    {
                        v0 = Avx.Max(v0, vmin);
                        v0 = Avx.Multiply(v0, Avx.ReciprocalSqrt(v0));
                        v0 = Avx.Add(v0, vd);
                        v0 = Avx.Max(v0, vmin);
                        v0 = Avx.Multiply(v0, v0);
                    }
                    else
                    {
                        v0 = Avx.Add(v0, vd);
                    }

                    Avx.Store(op, v0);
                    op += VectorAvx.Count;
                } while (ip <= ipe);
                ipe += VectorAvx.Count;
            }
            else if (ip <= ipe - VectorSse.Count)
            {
                var vthresh = Vector128.Create(threshold ? thresh : -1f);
                var vmsk    = Vector128.Create(0x7fffffff).AsSingle();
                var vamt    = Vector128.Create(amt);
                var vmin    = VectorSse.Zero;

                ipe -= VectorSse.Count;
                do
                {
                    var vd = Sse.Subtract(Sse.LoadVector128(yp), Sse.LoadVector128(bp));
                    yp += VectorSse.Count;
                    bp += VectorSse.Count;

                    if (threshold)
                    {
                        var sm = Sse.CompareGreaterThan(Sse.And(vd, vmsk), vthresh);
                        vd = Sse.And(vd, sm);
                    }
                    vd = Sse.Multiply(vd, vamt);

                    var v0 = Sse.LoadVector128(ip);
                    ip += VectorSse.Count;

                    if (gamma)
                    {
                        v0 = Sse.Max(v0, vmin);
                        v0 = Sse.Multiply(v0, Sse.ReciprocalSqrt(v0));
                        v0 = Sse.Add(v0, vd);
                        v0 = Sse.Max(v0, vmin);
                        v0 = Sse.Multiply(v0, v0);
                    }
                    else
                    {
                        v0 = Sse.Add(v0, vd);
                    }

                    Sse.Store(op, v0);
                    op += VectorSse.Count;
                } while (ip <= ipe);
                ipe += VectorSse.Count;
            }

            float fmin = VectorSse.Zero.ToScalar();

            while (ip < ipe)
            {
                float dif = *yp++ - *bp++;
                float c0  = *ip++;

                if (!threshold || Math.Abs(dif) > thresh)
                {
                    dif *= amt;

                    if (gamma)
                    {
                        c0  = MathUtil.MaxF(c0, fmin).Sqrt();
                        c0  = MathUtil.MaxF(c0 + dif, fmin);
                        c0 *= c0;
                    }
                    else
                    {
                        c0 += dif;
                    }
                }

                *op++ = c0;
            }
        }
Example #5
0
        unsafe void IConvolver.ConvolveSourceLine(byte *istart, byte *tstart, int cb, byte *mapxstart, int smapx, int smapy)
        {
            float *tp = (float *)tstart, tpe = (float *)(tstart + cb);
            float *pmapx   = (float *)mapxstart;
            int    kstride = smapx * channels;
            int    tstride = smapy * channels;
            int    vcnt    = smapx / Vector128 <float> .Count;

            while (tp < tpe)
            {
                int ix   = *(int *)pmapx++;
                int lcnt = vcnt;

                float *ip = (float *)istart + ix * channels;
                float *mp = pmapx;
                pmapx += kstride;

                Vector128 <float> av0;

                if (Avx.IsSupported && lcnt >= 2)
                {
                    var ax0 = Vector256 <float> .Zero;

                    for (; lcnt >= 2; lcnt -= 2)
                    {
                        var iv0 = Avx.LoadVector256(ip);
                        var iv1 = Avx.LoadVector256(ip + Vector256 <float> .Count);
                        var iv2 = Avx.LoadVector256(ip + Vector256 <float> .Count * 2);
                        var iv3 = Avx.LoadVector256(ip + Vector256 <float> .Count * 3);
                        ip += Vector256 <int> .Count * channels;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 2), iv2, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 3), iv3, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv2, Avx.LoadVector256(mp + Vector256 <float> .Count * 2)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv3, Avx.LoadVector256(mp + Vector256 <float> .Count * 3)));
                        }
                        mp += Vector256 <float> .Count * channels;
                    }

                    av0 = Sse.Add(ax0.GetLower(), ax0.GetUpper());
                }
                else
                {
                    av0 = Vector128 <float> .Zero;
                }

                for (; lcnt != 0; lcnt--)
                {
                    var iv0 = Sse.LoadVector128(ip);
                    var iv1 = Sse.LoadVector128(ip + Vector128 <float> .Count);
                    var iv2 = Sse.LoadVector128(ip + Vector128 <float> .Count * 2);
                    var iv3 = Sse.LoadVector128(ip + Vector128 <float> .Count * 3);
                    ip += Vector128 <float> .Count * channels;

                    if (Fma.IsSupported)
                    {
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp), iv0, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count), iv1, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count * 2), iv2, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count * 3), iv3, av0);
                    }
                    else
                    {
                        av0 = Sse.Add(av0, Sse.Multiply(iv0, Sse.LoadVector128(mp)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv1, Sse.LoadVector128(mp + Vector128 <float> .Count)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv2, Sse.LoadVector128(mp + Vector128 <float> .Count * 2)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv3, Sse.LoadVector128(mp + Vector128 <float> .Count * 3)));
                    }
                    mp += Vector128 <float> .Count * channels;
                }

                tp[0] = av0.ToScalar();
                tp[1] = Sse.Shuffle(av0, av0, 0b_11_10_01_01).ToScalar();
                tp[2] = Sse.UnpackHigh(av0, av0).ToScalar();
                tp[3] = Sse.Shuffle(av0, av0, 0b_11_10_01_11).ToScalar();
                tp   += tstride;
            }
        }
Example #6
0
        unsafe void IConvolver.WriteDestLine(byte *tstart, byte *ostart, int ox, int ow, byte *pmapy, int smapy)
        {
            float *op = (float *)ostart;
            int    xc = ox + ow, tstride = smapy;
            int    vcnt = smapy / Vector128 <float> .Count;

            while (ox < xc)
            {
                int lcnt = vcnt;

                float *tp = (float *)tstart + ox * tstride;
                float *mp = (float *)pmapy;

                Vector128 <float> av0;

                if (Avx.IsSupported && lcnt >= 2)
                {
                    var ax0 = Vector256 <float> .Zero;

                    for (; lcnt >= 4; lcnt -= 4)
                    {
                        var iv0 = Avx.LoadVector256(tp);
                        var iv1 = Avx.LoadVector256(tp + Vector256 <float> .Count);
                        tp += Vector256 <float> .Count * 2;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                        }
                        mp += Vector256 <float> .Count * 2;
                    }

                    if (lcnt >= 2)
                    {
                        lcnt -= 2;

                        var iv0 = Avx.LoadVector256(tp);
                        tp += Vector256 <float> .Count;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                        }

                        mp += Vector256 <float> .Count;
                    }

                    av0 = Sse.Add(ax0.GetLower(), ax0.GetUpper());
                }
                else
                {
                    av0 = Vector128 <float> .Zero;
                }

                for (; lcnt != 0; lcnt--)
                {
                    var iv0 = Sse.LoadVector128(tp);
                    tp += Vector128 <float> .Count;

                    if (Fma.IsSupported)
                    {
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp), iv0, av0);
                    }
                    else
                    {
                        av0 = Sse.Add(av0, Sse.Multiply(iv0, Sse.LoadVector128(mp)));
                    }

                    mp += Vector128 <float> .Count;
                }

                *op++ = av0.HorizontalAdd();
                ox++;
            }
        }
Example #7
0
        unsafe void IConvolver.WriteDestLine(byte *tstart, byte *ostart, int ox, int ow, byte *pmapy, int smapy)
        {
            float *op = (float *)ostart;
            int    xc = ox + ow, tstride = smapy * channels;
            int    vcnt = smapy / Vector128 <float> .Count;

            while (ox < xc)
            {
                int lcnt = vcnt;

                float *tp = (float *)tstart + ox * tstride;
                float *mp = (float *)pmapy;

                Vector128 <float> av0;

                if (Avx.IsSupported && lcnt >= 2)
                {
                    var ax0 = Vector256 <float> .Zero;

                    for (; lcnt >= 2; lcnt -= 2)
                    {
                        var iv0 = Avx.LoadVector256(tp);
                        var iv1 = Avx.LoadVector256(tp + Vector256 <float> .Count);
                        var iv2 = Avx.LoadVector256(tp + Vector256 <float> .Count * 2);
                        var iv3 = Avx.LoadVector256(tp + Vector256 <float> .Count * 3);
                        tp += Vector256 <int> .Count * channels;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 2), iv2, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 3), iv3, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv2, Avx.LoadVector256(mp + Vector256 <float> .Count * 2)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv3, Avx.LoadVector256(mp + Vector256 <float> .Count * 3)));
                        }
                        mp += Vector256 <float> .Count * channels;
                    }

                    av0 = Sse.Add(ax0.GetLower(), ax0.GetUpper());
                }
                else
                {
                    av0 = Vector128 <float> .Zero;
                }

                for (; lcnt != 0; lcnt--)
                {
                    var iv0 = Sse.LoadVector128(tp);
                    var iv1 = Sse.LoadVector128(tp + Vector128 <float> .Count);
                    var iv2 = Sse.LoadVector128(tp + Vector128 <float> .Count * 2);
                    var iv3 = Sse.LoadVector128(tp + Vector128 <float> .Count * 3);
                    tp += Vector128 <float> .Count * channels;

                    if (Fma.IsSupported)
                    {
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp), iv0, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count), iv1, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count * 2), iv2, av0);
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp + Vector128 <float> .Count * 3), iv3, av0);
                    }
                    else
                    {
                        av0 = Sse.Add(av0, Sse.Multiply(iv0, Sse.LoadVector128(mp)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv1, Sse.LoadVector128(mp + Vector128 <float> .Count)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv2, Sse.LoadVector128(mp + Vector128 <float> .Count * 2)));
                        av0 = Sse.Add(av0, Sse.Multiply(iv3, Sse.LoadVector128(mp + Vector128 <float> .Count * 3)));
                    }
                    mp += Vector128 <float> .Count * channels;
                }

                op[0] = av0.ToScalar();
                op[1] = Sse.Shuffle(av0, av0, 0b_11_10_01_01).ToScalar();
                op[2] = Sse.UnpackHigh(av0, av0).ToScalar();
                op[3] = Sse.Shuffle(av0, av0, 0b_11_10_01_11).ToScalar();
                op   += channels;
                ox++;
            }
        }
Example #8
0
        unsafe void IConvolver.ConvolveSourceLine(byte *istart, byte *tstart, int cb, byte *mapxstart, int smapx, int smapy)
        {
            float *tp = (float *)tstart, tpe = (float *)(tstart + cb);
            float *pmapx   = (float *)mapxstart;
            int    kstride = smapx;
            int    tstride = smapy;
            int    vcnt    = smapx / Vector128 <float> .Count;

            while (tp < tpe)
            {
                int ix   = *(int *)pmapx++;
                int lcnt = vcnt;

                float *ip = (float *)istart + ix;
                float *mp = pmapx;
                pmapx += kstride;

                Vector128 <float> av0;

                if (Avx.IsSupported && lcnt >= 2)
                {
                    var ax0 = Vector256 <float> .Zero;

                    for (; lcnt >= 8; lcnt -= 8)
                    {
                        var iv0 = Avx.LoadVector256(ip);
                        var iv1 = Avx.LoadVector256(ip + Vector256 <float> .Count);
                        var iv2 = Avx.LoadVector256(ip + Vector256 <float> .Count * 2);
                        var iv3 = Avx.LoadVector256(ip + Vector256 <float> .Count * 3);
                        ip += Vector256 <float> .Count * 4;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 2), iv2, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 3), iv3, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv2, Avx.LoadVector256(mp + Vector256 <float> .Count * 2)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv3, Avx.LoadVector256(mp + Vector256 <float> .Count * 3)));
                        }
                        mp += Vector256 <float> .Count * 4;
                    }

                    if (lcnt >= 6)
                    {
                        lcnt -= 6;

                        var iv0 = Avx.LoadVector256(ip);
                        var iv1 = Avx.LoadVector256(ip + Vector256 <float> .Count);
                        var iv2 = Avx.LoadVector256(ip + Vector256 <float> .Count * 2);
                        ip += Vector256 <float> .Count * 3;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count * 2), iv2, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv2, Avx.LoadVector256(mp + Vector256 <float> .Count * 2)));
                        }
                        mp += Vector256 <float> .Count * 3;
                    }
                    else if (lcnt >= 4)
                    {
                        lcnt -= 4;

                        var iv0 = Avx.LoadVector256(ip);
                        var iv1 = Avx.LoadVector256(ip + Vector256 <float> .Count);
                        ip += Vector256 <float> .Count * 2;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp + Vector256 <float> .Count), iv1, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv1, Avx.LoadVector256(mp + Vector256 <float> .Count)));
                        }
                        mp += Vector256 <float> .Count * 2;
                    }
                    else if (lcnt >= 2)
                    {
                        lcnt -= 2;

                        var iv0 = Avx.LoadVector256(ip);
                        ip += Vector256 <float> .Count;

                        if (Fma.IsSupported)
                        {
                            ax0 = Fma.MultiplyAdd(Avx.LoadVector256(mp), iv0, ax0);
                        }
                        else
                        {
                            ax0 = Avx.Add(ax0, Avx.Multiply(iv0, Avx.LoadVector256(mp)));
                        }

                        mp += Vector256 <float> .Count;
                    }

                    av0 = Sse.Add(ax0.GetLower(), ax0.GetUpper());
                }
                else
                {
                    av0 = Vector128 <float> .Zero;
                }

                for (; lcnt != 0; lcnt--)
                {
                    var iv0 = Sse.LoadVector128(ip);
                    ip += Vector128 <float> .Count;

                    if (Fma.IsSupported)
                    {
                        av0 = Fma.MultiplyAdd(Sse.LoadVector128(mp), iv0, av0);
                    }
                    else
                    {
                        av0 = Sse.Add(av0, Sse.Multiply(iv0, Sse.LoadVector128(mp)));
                    }

                    mp += Vector128 <float> .Count;
                }

                tp[0] = av0.HorizontalAdd();
                tp   += tstride;
            }
        }
Example #9
0
        public static unsafe float GetScribnerBoardFeetPerAcre(Trees trees)
        {
            // for now, assume all trees are of the same species
            if (trees.Species != FiaCode.PseudotsugaMenziesii)
            {
                throw new NotSupportedException();
            }
            if (trees.Units != Units.English)
            {
                throw new NotSupportedException();
            }

            // Douglas-fir
            #if DEBUG
            Vector128 <float> v6p8 = AvxExtensions.BroadcastScalarToVector128(6.8F);
            Vector128 <float> v10k = AvxExtensions.BroadcastScalarToVector128(10.0F * 1000.0F);
            #endif

            // constants
            Vector128 <float> forestersEnglish = AvxExtensions.BroadcastScalarToVector128(Constant.ForestersEnglish);
            Vector128 <float> one = AvxExtensions.BroadcastScalarToVector128(1.0F);
            Vector128 <float> six = AvxExtensions.BroadcastScalarToVector128(6.0F);

            Vector128 <float> vm3p21809   = AvxExtensions.BroadcastScalarToVector128(-3.21809F); // b4
            Vector128 <float> v0p04948    = AvxExtensions.BroadcastScalarToVector128(0.04948F);
            Vector128 <float> vm0p15664   = AvxExtensions.BroadcastScalarToVector128(-0.15664F);
            Vector128 <float> v2p02132    = AvxExtensions.BroadcastScalarToVector128(2.02132F);
            Vector128 <float> v1p63408    = AvxExtensions.BroadcastScalarToVector128(1.63408F);
            Vector128 <float> vm0p16184   = AvxExtensions.BroadcastScalarToVector128(-0.16184F);
            Vector128 <float> v1p033      = AvxExtensions.BroadcastScalarToVector128(1.033F);
            Vector128 <float> v1p382937   = AvxExtensions.BroadcastScalarToVector128(1.382937F);
            Vector128 <float> vm0p4015292 = AvxExtensions.BroadcastScalarToVector128(-0.4015292F);
            Vector128 <float> v0p087266   = AvxExtensions.BroadcastScalarToVector128(0.087266F);
            Vector128 <float> vm0p174533  = AvxExtensions.BroadcastScalarToVector128(-0.174533F);

            Vector128 <float> vm0p6896598794 = AvxExtensions.BroadcastScalarToVector128(-0.6896598794F); // rc6-rs632
            Vector128 <float> v0p993         = AvxExtensions.BroadcastScalarToVector128(0.993F);
            Vector128 <float> v0p174439      = AvxExtensions.BroadcastScalarToVector128(0.174439F);
            Vector128 <float> v0p117594      = AvxExtensions.BroadcastScalarToVector128(0.117594F);
            Vector128 <float> vm8p210585     = AvxExtensions.BroadcastScalarToVector128(-8.210585F);
            Vector128 <float> v0p236693      = AvxExtensions.BroadcastScalarToVector128(0.236693F);
            Vector128 <float> v0p00001345    = AvxExtensions.BroadcastScalarToVector128(0.00001345F);
            Vector128 <float> v0p00001937    = AvxExtensions.BroadcastScalarToVector128(0.00001937F);
            Vector128 <float> v1p001491      = AvxExtensions.BroadcastScalarToVector128(1.001491F);
            Vector128 <float> vm6p924097     = AvxExtensions.BroadcastScalarToVector128(-6.924097F);
            Vector128 <float> v0p912733      = AvxExtensions.BroadcastScalarToVector128(0.912733F);
            Vector128 <float> v0p00001351    = AvxExtensions.BroadcastScalarToVector128(0.00001351F);

            fixed(float *dbh = &trees.Dbh[0], expansionFactors = &trees.LiveExpansionFactor[0], height = &trees.Height[0])
            {
                Vector128 <float> standBoardFeetPerAcre = Vector128 <float> .Zero;

                for (int treeIndex = 0; treeIndex < trees.Count; treeIndex += Constant.Simd128x4.Width)
                {
                    Vector128 <float> dbhInInches  = Avx.LoadVector128(dbh + treeIndex);
                    Vector128 <float> heightInFeet = Avx.LoadVector128(height + treeIndex);

                    Vector128 <float> logDbhInInches  = MathV.Log10(dbhInInches);
                    Vector128 <float> logHeightInFeet = MathV.Log10(heightInFeet);
                    // FiaCode.PseudotsugaMenziesii => -3.21809F + 0.04948F * logHeightInFeet * logDbhInInches - 0.15664F * logDbhInInches * logDbhInInches +
                    //                                  2.02132F * logDbhInInches + 1.63408F * logHeightInFeet - 0.16184F * logHeightInFeet * logHeightInFeet,
                    Vector128 <float> cvtsl = Avx.Add(vm3p21809, Avx.Multiply(v0p04948, Avx.Multiply(logHeightInFeet, logDbhInInches)));
                    cvtsl = Avx.Add(cvtsl, Avx.Multiply(vm0p15664, Avx.Multiply(logDbhInInches, logDbhInInches)));
                    cvtsl = Avx.Add(cvtsl, Avx.Multiply(v2p02132, logDbhInInches));
                    cvtsl = Avx.Add(cvtsl, Avx.Multiply(v1p63408, logHeightInFeet));
                    cvtsl = Avx.Add(cvtsl, Avx.Multiply(vm0p16184, Avx.Multiply(logHeightInFeet, logHeightInFeet)));
                    Vector128 <float> cubicFeet = MathV.Exp10(cvtsl);

                    Vector128 <float> dbhSquared            = Avx.Multiply(dbhInInches, dbhInInches); // could be consolidated by merging other scaling constants with Forester's constant for basal area
                    Vector128 <float> basalAreaInSquareFeet = Avx.Multiply(forestersEnglish, dbhSquared);
                    // b4 = cubicFeet / (1.033F * (1.0F + 1.382937F * MathV.Exp(-4.015292F * dbhInInches / 10.0F)) * (basalAreaInSquareFeet + 0.087266F) - 0.174533F);
                    Vector128 <float> b4 = Avx.Divide(cubicFeet, Avx.Add(Avx.Multiply(v1p033,
                                                                                      Avx.Multiply(Avx.Add(one, Avx.Multiply(v1p382937,
                                                                                                                             MathV.Exp(Avx.Multiply(vm0p4015292,
                                                                                                                                                    dbhInInches)))),
                                                                                                   Avx.Add(basalAreaInSquareFeet, v0p087266))),
                                                                         vm0p174533));
                    Vector128 <float> cv4 = Avx.Multiply(b4, Avx.Subtract(basalAreaInSquareFeet, v0p087266));

                    // conversion to Scribner volumes for 32 foot trees
                    // Waddell 2014:32
                    // rc6 = 0.993F * (1.0F - MathF.Pow(0.62F, dbhInInches - 6.0F));
                    Vector128 <float> rc6   = Avx.Multiply(v0p993, Avx.Subtract(one, MathV.Exp(Avx.Multiply(vm0p6896598794, Avx.Subtract(dbhInInches, six))))); // log2(0.62) = -0.6896598794
                    Vector128 <float> cv6   = Avx.Multiply(rc6, cv4);
                    Vector128 <float> logB4 = MathV.Log10(b4);
                    // float rs616 = MathF.Pow(10.0F, 0.174439F + 0.117594F * logDbhInInches * logB4 - 8.210585F / (dbhInInches * dbhInInches) + 0.236693F * logB4 - 0.00001345F * b4 * b4 - 0.00001937F * dbhInInches * dbhInInches);
                    Vector128 <float> rs616l = Avx.Add(v0p174439, Avx.Multiply(v0p117594, Avx.Multiply(logDbhInInches, logB4)));
                    rs616l = Avx.Add(rs616l, Avx.Divide(vm8p210585, dbhSquared));
                    rs616l = Avx.Add(rs616l, Avx.Multiply(v0p236693, logB4));
                    rs616l = Avx.Subtract(rs616l, Avx.Multiply(v0p00001345, Avx.Multiply(b4, b4)));
                    rs616l = Avx.Subtract(rs616l, Avx.Multiply(v0p00001937, dbhSquared));
                    Vector128 <float> rs616 = MathV.Exp10(rs616l);
                    Vector128 <float> sv616 = Avx.Multiply(rs616, cv6); // Scribner board foot volume to a 6 inch top for 16 foot logs
                    // float rs632 = 1.001491F - 6.924097F / tarif + 0.00001351F * dbhInInches * dbhInInches;
                    Vector128 <float> rs632 = Avx.Add(v1p001491, Avx.Divide(vm6p924097, Avx.Multiply(v0p912733, b4)));
                    rs632 = Avx.Add(rs632, Avx.Multiply(v0p00001351, dbhSquared));
                    Vector128 <float> zeroVolumeMask = Avx.CompareLessThanOrEqual(dbhInInches, six);
                    Vector128 <float> sv632          = Avx.Multiply(rs632, sv616); // Scribner board foot volume to a 6 inch top for 32 foot logs
                    sv632 = Avx.BlendVariable(sv632, Vector128 <float> .Zero, zeroVolumeMask);

                    #if DEBUG
                    DebugV.Assert(Avx.CompareGreaterThanOrEqual(Avx.BlendVariable(rc6, Vector128 <float> .Zero, zeroVolumeMask), Vector128 <float> .Zero));
                    DebugV.Assert(Avx.CompareLessThanOrEqual(rc6, one));
                    DebugV.Assert(Avx.CompareGreaterThanOrEqual(Avx.BlendVariable(rs616, one, zeroVolumeMask), one));
                    DebugV.Assert(Avx.CompareLessThanOrEqual(Avx.BlendVariable(rs616, Vector128 <float> .Zero, zeroVolumeMask), v6p8));
                    DebugV.Assert(Avx.CompareGreaterThanOrEqual(Avx.BlendVariable(rs632, Vector128 <float> .Zero, zeroVolumeMask), Vector128 <float> .Zero));
                    DebugV.Assert(Avx.CompareLessThanOrEqual(Avx.BlendVariable(rs632, Vector128 <float> .Zero, zeroVolumeMask), one));
                    DebugV.Assert(Avx.CompareGreaterThanOrEqual(Avx.BlendVariable(sv632, Vector128 <float> .Zero, zeroVolumeMask), Vector128 <float> .Zero));
                    DebugV.Assert(Avx.CompareLessThanOrEqual(Avx.BlendVariable(sv632, Vector128 <float> .Zero, zeroVolumeMask), v10k));
                    #endif

                    Vector128 <float> expansionFactor = Avx.LoadVector128(expansionFactors + treeIndex);
                    standBoardFeetPerAcre = Avx.Add(standBoardFeetPerAcre, Avx.Multiply(expansionFactor, sv632));
                }

                standBoardFeetPerAcre = Avx.HorizontalAdd(standBoardFeetPerAcre, standBoardFeetPerAcre);
                standBoardFeetPerAcre = Avx.HorizontalAdd(standBoardFeetPerAcre, standBoardFeetPerAcre);
                return(standBoardFeetPerAcre.ToScalar());
            }
        }
Example #10
0
        public static unsafe ComplexFloat[] Kernel32(ComplexFloat[] i, ref ComplexFloat[][] omegas)
        {
            ComplexFloat[] result = new ComplexFloat[32];
            ComplexFloat[] tmp    = new ComplexFloat[48];

            ComplexFloat ami = i[0] - i[8];
            ComplexFloat api = i[0] + i[8];
            ComplexFloat fmn = i[5] - i[13];
            ComplexFloat fpn = i[5] + i[13];

            ComplexFloat xami = i[16] - i[24];
            ComplexFloat xapi = i[16] + i[24];
            ComplexFloat xfmn = i[21] - i[29];
            ComplexFloat xfpn = i[21] + i[29];

            tmp[0] = api + i[2] + i[4] + i[6] + i[10] + i[12] + i[14];
            tmp[1] = ami + (i[2] - i[10] + (i[6] - i[14]).TimesMinusI()) * omegas[3][1] + (i[4] - i[12]).TimesMinusI();
            tmp[2] = api - i[4] - i[12] + (i[2] - i[6] + i[10] - i[14]).TimesMinusI();
            tmp[3] = ami - (i[4] - i[12]).TimesMinusI() - (i[10] - i[2] + (i[6] - i[14]).TimesMinusI()) * omegas[3][3];
            tmp[4] = api - i[2] + i[4] - i[6] - i[10] + i[12] - i[14];
            tmp[5] = ami - (i[2] - i[10] + (i[6] - i[14]).TimesMinusI()) * omegas[3][1] + (i[4] - i[12]).TimesMinusI();
            tmp[6] = api - i[4] - i[12] - (i[2] - i[6] + i[10] - i[14]).TimesMinusI();
            tmp[7] = ami - (i[4] - i[12]).TimesMinusI() + (i[10] - i[2] + (i[6] - i[14]).TimesMinusI()).TimesMinusI();

            tmp[8]  = i[1] + i[3] + fpn + i[7] + i[9] + i[11] + i[15];
            tmp[9]  = omegas[4][1] * (i[1] - i[9] + (i[3] - i[11] + (i[7] - i[15]).TimesMinusI()) * omegas[3][1] + (fmn).TimesMinusI());
            tmp[10] = omegas[4][2] * ((i[3] - i[7] + i[11] - i[15]).TimesMinusI() + i[1] - fpn + i[9]);
            tmp[11] = omegas[4][3] * (omegas[3][3] * (i[11] - i[3] + (i[7] - i[15]).TimesMinusI()) - i[1] + i[9] + (fmn).TimesMinusI());
            tmp[12] = (i[1] - i[3] + fpn - i[7] + i[9] - i[11] - i[15]).TimesMinusI();
            tmp[13] = (i[1] - i[9] - (i[3] - i[11] + (i[7] - i[15]).TimesMinusI()) * omegas[3][1] + (fmn).TimesMinusI()) * omegas[4][5];
            tmp[14] = omegas[4][6] * ((i[3] - i[7] + i[11] - i[15]).TimesMinusI() - i[1] + fpn - i[9]);
            tmp[15] = omegas[4][7] * ((i[11] - i[3] + (i[7] - i[15]).TimesMinusI()).TimesMinusI() + i[1] - i[9] - (fmn).TimesMinusI());

            tmp[16] = xapi + i[18] + i[20] + i[22] + i[28] + i[26] + i[30];
            tmp[17] = xami + (i[18] - i[26] + (i[22] - i[30]).TimesMinusI()) * omegas[3][1] + (i[20] - i[28]).TimesMinusI();
            tmp[18] = xapi - i[20] - i[28] + (i[18] - i[22] + i[26] - i[30]).TimesMinusI();
            tmp[19] = xami - (i[20] - i[28]).TimesMinusI() - (i[26] - i[18] + (i[22] - i[30]).TimesMinusI()) * omegas[3][3];
            tmp[20] = xapi - i[28] + i[20] - i[22] - i[26] + i[28] - i[30];
            tmp[21] = xami - (i[28] - i[26] + (i[22] - i[30]).TimesMinusI()) * omegas[3][1] + (i[20] - i[22]).TimesMinusI();
            tmp[22] = xapi - i[20] - i[28] - (i[18] - i[22] + i[26] - i[30]).TimesMinusI();
            tmp[23] = xami - (i[20] - i[28]).TimesMinusI() + (i[26] - i[18] + (i[22] - i[30]).TimesMinusI()).TimesMinusI();

            tmp[24] = i[17] + i[19] + xfpn + i[23] + i[25] + i[27] + i[31];
            tmp[25] = omegas[4][1] * (i[17] - i[25] + (i[19] - i[27] + (i[23] - i[31]).TimesMinusI()) * omegas[3][1] + (xfmn).TimesMinusI());
            tmp[26] = omegas[4][2] * ((i[19] - i[23] + i[27] - i[31]).TimesMinusI() + i[17] - xfpn + i[25]);
            tmp[27] = omegas[4][3] * (omegas[3][3] * (i[27] - i[19] + (i[23] - i[31]).TimesMinusI()) - i[17] + i[25] + (xfmn).TimesMinusI());
            tmp[28] = (i[17] - i[19] + xfpn - i[23] + i[25] - i[27] - i[31]).TimesMinusI();
            tmp[29] = (i[17] - i[25] - (i[19] - i[27] + (i[23] - i[31]).TimesMinusI()) * omegas[3][1] + (xfmn).TimesMinusI()) * omegas[4][5];
            tmp[30] = omegas[4][6] * ((i[19] - i[23] + i[27] - i[31]).TimesMinusI() - i[17] + xfpn - i[25]);
            tmp[31] = omegas[4][7] * ((i[27] - i[19] + (i[23] - i[31]).TimesMinusI()).TimesMinusI() + i[17] - i[25] - (xfmn).TimesMinusI());


            //32 complex floats = 64 floats
            //Divided into 4 parts A, B, C, D = each containing 8 complex floats, so 16 floats
            //AVX takes 8 floats at once, so will calculate in halves of those parts
            //Tmp will ocntain 6 octets

            fixed(ComplexFloat *entry = result, om5 = omegas[5], t = tmp)
            {
                Vector256 <float> a;
                Vector256 <float> b;
                Vector256 <float> bSwap;
                Vector256 <float> aIm;
                Vector256 <float> aRe;
                Vector256 <float> aIM_bSwap;

                float *partA    = (float *)entry;
                float *partB    = partA + 16;
                float *partC    = partA + 32;
                float *partD    = partA + 48;
                float *omPart1  = (float *)om5;
                float *omPart2  = omPart1 + 16;
                float *tmpPart1 = (float *)t;
                float *tmpPart2 = tmpPart1 + 16;
                float *tmpPart3 = tmpPart1 + 32;
                float *tmpPart4 = tmpPart1 + 48;
                float *tmpPart5 = tmpPart1 + 64;
                float *tmpPart6 = tmpPart1 + 80;

                //Summing up result

                Avx2.Store(partA, Avx2.Add(Avx2.LoadVector256(tmpPart1), Avx2.LoadVector256(tmpPart2)));
                Avx2.Store(partA + 8, Avx2.Add(Avx2.LoadVector256(tmpPart1 + 8), Avx2.LoadVector256(tmpPart2 + 8)));
                Avx2.Store(partB, Avx2.Subtract(Avx2.LoadVector256(tmpPart1), Avx2.LoadVector256(tmpPart2)));
                Avx2.Store(partB + 8, Avx2.Subtract(Avx2.LoadVector256(tmpPart1 + 8), Avx2.LoadVector256(tmpPart2 + 8)));

                Avx2.Store(partC, Avx2.Add(Avx2.LoadVector256(tmpPart3), Avx2.LoadVector256(tmpPart4)));
                Avx2.Store(partC + 8, Avx2.Add(Avx2.LoadVector256(tmpPart3 + 8), Avx2.LoadVector256(tmpPart4 + 8)));
                Avx2.Store(partD, Avx2.Subtract(Avx2.LoadVector256(tmpPart3), Avx2.LoadVector256(tmpPart4)));
                Avx2.Store(partD + 8, Avx2.Subtract(Avx2.LoadVector256(tmpPart3 + 8), Avx2.LoadVector256(tmpPart4 + 8)));



                //-------------------------------------------------------------------------------------------------------------

                //First part of each 8 complex part

                //Tmp[0] = A + B
                Avx2.Store(tmpPart1, Avx2.Add(Avx2.LoadVector256(partA), Avx2.LoadVector256(partB)));
                //Tmp[1] = A - B
                Avx2.Store(tmpPart2, Avx2.Subtract(Avx2.LoadVector256(partA), Avx2.LoadVector256(partB)));
                //Tmp[2] = C + D
                Avx2.Store(tmpPart3, Avx2.Add(Avx2.LoadVector256(partC), Avx2.LoadVector256(partD)));
                //Tmp[3] = C - D
                Avx2.Store(tmpPart4, Avx2.Subtract(Avx2.LoadVector256(partC), Avx2.LoadVector256(partD)));

                //Complex multiplication based on: https://www.researchgate.net/figure/Vectorized-complex-multiplication-using-AVX-2_fig2_337532904

                //Tmp[4] = omega * (C+D)
                a         = Avx2.LoadVector256(tmpPart3);
                b         = Avx2.LoadVector256(omPart1);
                bSwap     = Avx2.Shuffle(b, b, imm8bShuffle);
                aIm       = Avx2.Shuffle(a, a, imm8aImShuffle);
                aRe       = Avx2.Shuffle(a, a, imm8aReShuffle);
                aIM_bSwap = Avx.Multiply(aIm, bSwap);
                Avx2.Store(tmpPart5, Fma.MultiplyAddSubtract(aRe, b, aIM_bSwap));

                //Tmp[4] = omega * (C-D)
                a         = Avx2.LoadVector256(tmpPart4);
                b         = Avx2.LoadVector256(omPart2);
                bSwap     = Avx2.Shuffle(b, b, imm8bShuffle);
                aIm       = Avx2.Shuffle(a, a, imm8aImShuffle);
                aRe       = Avx2.Shuffle(a, a, imm8aReShuffle);
                aIM_bSwap = Avx.Multiply(aIm, bSwap);
                Avx2.Store(tmpPart6, Fma.MultiplyAddSubtract(aRe, b, aIM_bSwap));

                //(A+B) + (C+D)
                Avx2.Store(partA, Avx.Add(Avx.LoadVector256(tmpPart1), Avx.LoadVector256(tmpPart3)));
                //(A-B) + (C-D)
                Avx2.Store(partB, Avx.Add(Avx.LoadVector256(tmpPart2), Avx.LoadVector256(tmpPart4)));
                //(A+B) + omega*(C+D)
                Avx2.Store(partC, Avx.Add(Avx.LoadVector256(tmpPart1), Avx.LoadVector256(tmpPart5)));
                //(A-B) + omega*(C-D)
                Avx2.Store(partD, Avx.Add(Avx.LoadVector256(tmpPart2), Avx.LoadVector256(tmpPart6)));

                //--------------------------------------------------------------------------------------------------------------

                //Second part of each 8 complex part

                //Tmp[0] = A + B
                Avx2.Store(tmpPart1, Avx2.Add(Avx2.LoadVector256(partA + 8), Avx2.LoadVector256(partB + 8)));
                //Tmp[1] = A - B
                Avx2.Store(tmpPart2, Avx2.Subtract(Avx2.LoadVector256(partA + 8), Avx2.LoadVector256(partB + 8)));
                //Tmp[2] = C + D
                Avx2.Store(tmpPart3, Avx2.Add(Avx2.LoadVector256(partC + 8), Avx2.LoadVector256(partD + 8)));
                //Tmp[2] = C - D
                Avx2.Store(tmpPart4, Avx2.Subtract(Avx2.LoadVector256(partC + 8), Avx2.LoadVector256(partD + 8)));

                //Complex multiplication based on: https://www.researchgate.net/figure/Vectorized-complex-multiplication-using-AVX-2_fig2_337532904

                //Tmp[4] = omega * (C+D)
                a         = Avx2.LoadVector256(tmpPart3);
                b         = Avx2.LoadVector256(omPart1 + 8);
                bSwap     = Avx2.Shuffle(b, b, imm8bShuffle);
                aIm       = Avx2.Shuffle(a, a, imm8aImShuffle);
                aRe       = Avx2.Shuffle(a, a, imm8aReShuffle);
                aIM_bSwap = Avx.Multiply(aIm, bSwap);
                Avx2.Store(tmpPart5, Fma.MultiplyAddSubtract(aRe, b, aIM_bSwap));

                //Tmp[4] = omega * (C-D)
                a         = Avx2.LoadVector256(tmpPart4);
                b         = Avx2.LoadVector256(omPart2 + 8);
                bSwap     = Avx2.Shuffle(b, b, imm8bShuffle);
                aIm       = Avx2.Shuffle(a, a, imm8aImShuffle);
                aRe       = Avx2.Shuffle(a, a, imm8aReShuffle);
                aIM_bSwap = Avx.Multiply(aIm, bSwap);
                Avx2.Store(tmpPart6, Fma.MultiplyAddSubtract(aRe, b, aIM_bSwap));

                //(A+B) + (C+D)
                Avx2.Store(partA + 8, Avx.Add(Avx.LoadVector256(tmpPart1), Avx.LoadVector256(tmpPart3)));
                //(A-B) + (C-D)
                Avx2.Store(partB + 8, Avx.Add(Avx.LoadVector256(tmpPart2), Avx.LoadVector256(tmpPart4)));
                //(A+B) + omega*(C+D)
                Avx2.Store(partC + 8, Avx.Add(Avx.LoadVector256(tmpPart1), Avx.LoadVector256(tmpPart5)));
                //(A-B) + omega*(C-D)
                Avx2.Store(partD + 8, Avx.Add(Avx.LoadVector256(tmpPart2), Avx.LoadVector256(tmpPart6)));
            }

            return(result);

            //ComplexFloat[] result = new ComplexFloat[32];
            //ArraySegment<ComplexFloat> partA = new ArraySegment<ComplexFloat>(i, 0, 16);
            //ArraySegment<ComplexFloat> partB = new ArraySegment<ComplexFloat>(i, 16, 16);
            //Kernel16(partA.ToArray(), ref omegas).CopyTo(result, 0);
            //Kernel16(partA.ToArray(), ref omegas).CopyTo(result, 16);
            //return result;
        }
Example #11
0
        public static unsafe RtMatrix operator *(RtMatrix value1, RtMatrix value2)
        {
            if (Avx2.IsSupported && useIntrinsics)
            {
                var row = Avx.LoadVector256(&value1.M11);
                Avx.Store(&value1.M11,
                          Avx.Add(Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0x00), Avx.LoadVector256(&value2.M11)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0x55), Avx.LoadVector256(&value2.M21))),
                                  Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0xAA), Avx.LoadVector256(&value2.M31)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0xFF), Avx.LoadVector256(&value2.M41)))));

                // 0x00 is _MM_SHUFFLE(0,0,0,0), 0x55 is _MM_SHUFFLE(1,1,1,1), etc.
                // TODO: Replace with a method once it's added to the API.

                row = Avx.LoadVector256(&value1.M21);
                Avx.Store(&value1.M21,
                          Avx.Add(Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0x00), Avx.LoadVector256(&value2.M11)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0x55), Avx.LoadVector256(&value2.M21))),
                                  Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0xAA), Avx.LoadVector256(&value2.M31)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0xFF), Avx.LoadVector256(&value2.M41)))));

                row = Avx.LoadVector256(&value1.M31);
                Avx.Store(&value1.M31,
                          Avx.Add(Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0x00), Avx.LoadVector256(&value2.M11)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0x55), Avx.LoadVector256(&value2.M21))),
                                  Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0xAA), Avx.LoadVector256(&value2.M31)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0xFF), Avx.LoadVector256(&value2.M41)))));

                row = Avx.LoadVector256(&value1.M41);
                Avx.Store(&value1.M41,
                          Avx.Add(Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0x00), Avx.LoadVector256(&value2.M11)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0x55), Avx.LoadVector256(&value2.M21))),
                                  Avx.Add(Avx.Multiply(Avx2.Permute4x64(row, 0xAA), Avx.LoadVector256(&value2.M31)),
                                          Avx.Multiply(Avx2.Permute4x64(row, 0xFF), Avx.LoadVector256(&value2.M41)))));
                return(value1);
            }

            RtMatrix m;

            // First row
            m.M11 = value1.M11 * value2.M11 + value1.M12 * value2.M21 + value1.M13 * value2.M31 + value1.M14 * value2.M41;
            m.M12 = value1.M11 * value2.M12 + value1.M12 * value2.M22 + value1.M13 * value2.M32 + value1.M14 * value2.M42;
            m.M13 = value1.M11 * value2.M13 + value1.M12 * value2.M23 + value1.M13 * value2.M33 + value1.M14 * value2.M43;
            m.M14 = value1.M11 * value2.M14 + value1.M12 * value2.M24 + value1.M13 * value2.M34 + value1.M14 * value2.M44;

            // Second row
            m.M21 = value1.M21 * value2.M11 + value1.M22 * value2.M21 + value1.M23 * value2.M31 + value1.M24 * value2.M41;
            m.M22 = value1.M21 * value2.M12 + value1.M22 * value2.M22 + value1.M23 * value2.M32 + value1.M24 * value2.M42;
            m.M23 = value1.M21 * value2.M13 + value1.M22 * value2.M23 + value1.M23 * value2.M33 + value1.M24 * value2.M43;
            m.M24 = value1.M21 * value2.M14 + value1.M22 * value2.M24 + value1.M23 * value2.M34 + value1.M24 * value2.M44;

            // Third row
            m.M31 = value1.M31 * value2.M11 + value1.M32 * value2.M21 + value1.M33 * value2.M31 + value1.M34 * value2.M41;
            m.M32 = value1.M31 * value2.M12 + value1.M32 * value2.M22 + value1.M33 * value2.M32 + value1.M34 * value2.M42;
            m.M33 = value1.M31 * value2.M13 + value1.M32 * value2.M23 + value1.M33 * value2.M33 + value1.M34 * value2.M43;
            m.M34 = value1.M31 * value2.M14 + value1.M32 * value2.M24 + value1.M33 * value2.M34 + value1.M34 * value2.M44;

            // Fourth row
            m.M41 = value1.M41 * value2.M11 + value1.M42 * value2.M21 + value1.M43 * value2.M31 + value1.M44 * value2.M41;
            m.M42 = value1.M41 * value2.M12 + value1.M42 * value2.M22 + value1.M43 * value2.M32 + value1.M44 * value2.M42;
            m.M43 = value1.M41 * value2.M13 + value1.M42 * value2.M23 + value1.M43 * value2.M33 + value1.M44 * value2.M43;
            m.M44 = value1.M41 * value2.M14 + value1.M42 * value2.M24 + value1.M43 * value2.M34 + value1.M44 * value2.M44;

            return(m);
        }
    private static unsafe double[] BilinearInterpol_AVX(
        double[] x,
        double[] A,
        double minXA,
        double maxXA,
        double[] B,
        double minXB,
        double maxXB,
        double weightB)
    {
        double[] z = new double[outputVectorSize];

        fixed(double *pX = &x[0], pA = &A[0], pB = &B[0], pZ = &z[0])
        {
            Vector256 <double> vWeightB = Vector256.Create(weightB);
            Vector256 <double> vWeightA = Vector256.Create(1 - weightB);

            Vector256 <double> vMinXA = Vector256.Create(minXA);
            Vector256 <double> vMaxXA = Vector256.Create(maxXA);
            Vector256 <double> vMinXB = Vector256.Create(minXB);
            Vector256 <double> vMaxXB = Vector256.Create(maxXB);

            double             deltaA  = (maxXA - minXA) / (double)(A.Length - 1);
            double             deltaB  = (maxXB - minXB) / (double)(B.Length - 1);
            Vector256 <double> vDeltaA = Vector256.Create(deltaA);
            Vector256 <double> vDeltaB = Vector256.Create(deltaB);

            double             invDeltaA  = 1.0 / deltaA;
            double             invDeltaB  = 1.0 / deltaB;
            Vector256 <double> vInvDeltaA = Vector256.Create(invDeltaA);
            Vector256 <double> vInvDeltaB = Vector256.Create(invDeltaB);

            Vector128 <int> ALengthMinusOne = Vector128.Create(A.Length - 1);
            Vector128 <int> BLengthMinusOne = Vector128.Create(B.Length - 1);
            Vector128 <int> One             = Vector128.Create(1);

            for (var i = 0; i < x.Length; i += Vector256 <double> .Count)
            {
                Vector256 <double> currentX = Avx.LoadVector256(pX + i);

                // Determine the largest a, such that A[i] = f(xA) and xA <= x[i].
                // This involves casting from double to int; here we use a Vector conversion.
                Vector256 <double> aDouble = Avx.Multiply(Avx.Subtract(currentX, vMinXA), vInvDeltaA);
                Vector128 <int>    a       = Avx.ConvertToVector128Int32WithTruncation(aDouble);
                a = Sse41.Min(Sse41.Max(a, Vector128 <int> .Zero), ALengthMinusOne);
                Vector128 <int> aPlusOne = Sse41.Min(Sse2.Add(a, One), ALengthMinusOne);

                // Now, get the reference input, xA, for our index a.
                // This involves casting from  int to double.
                Vector256 <double> xA = Avx.Add(Avx.Multiply(Avx.ConvertToVector256Double(a), vDeltaA), vMinXA);

                // Now, compute the lambda for our A reference point.
                Vector256 <double> currentXNormA = Avx.Max(vMinXA, Avx.Min(currentX, vMaxXA));
                Vector256 <double> lambdaA       = Avx.Multiply(Avx.Subtract(currentXNormA, xA), vInvDeltaA);

                // Now, we need to load up our reference points using Vector Gather operations.
                Vector256 <double> AVector        = Avx2.GatherVector256(pA, a, 8);
                Vector256 <double> AVectorPlusOne = Avx2.GatherVector256(pA, aPlusOne, 8);

                // Now, do the all of the above for our B reference point.
                Vector256 <double> bDouble = Avx.Multiply(Avx.Subtract(currentX, vMinXB), vInvDeltaB);
                Vector128 <int>    b       = Avx.ConvertToVector128Int32WithTruncation(bDouble);
                b = Sse41.Min(Sse41.Max(b, Vector128 <int> .Zero), BLengthMinusOne);
                Vector128 <int> bPlusOne = Sse41.Min(Sse2.Add(b, One), BLengthMinusOne);

                Vector256 <double> xB            = Avx.Add(Avx.Multiply(Avx.ConvertToVector256Double(b), vDeltaB), vMinXB);
                Vector256 <double> currentXNormB = Avx.Max(vMinXB, Avx.Min(currentX, vMaxXB));
                Vector256 <double> lambdaB       = Avx.Multiply(Avx.Subtract(currentXNormB, xB), vInvDeltaB);

                Vector256 <double> BVector        = Avx2.GatherVector256(pB, b, 8);
                Vector256 <double> BVectorPlusOne = Avx2.GatherVector256(pB, bPlusOne, 8);

                Vector256 <double> newZ = Avx.Add(Avx.Multiply(vWeightA, Avx.Add(AVector, Avx.Multiply(lambdaA, Avx.Subtract(AVectorPlusOne, AVector)))),
                                                  Avx.Multiply(vWeightB, Avx.Add(BVector, Avx.Multiply(lambdaB, Avx.Subtract(BVectorPlusOne, BVector)))));
                Avx.Store(pZ + i, newZ);
            }
        }

        return(z);
    }
Example #13
0
        public Intro()
        {
            var middleVector = Vector128.Create(1.0f);                      // middleVector = <1,1,1,1>

            middleVector = Vector128.CreateScalar(-1.0f);                   // middleVector = <-1,0,0,0>
            var floatBytes = Vector64.AsByte(Vector64.Create(1.0f, -1.0f)); // floatBytes = <0, 0, 128, 63, 0, 0, 128, 191>

            if (Avx.IsSupported)
            {
                var left  = Vector256.Create(-2.5f);                     // <-2.5, -2.5, -2.5, -2.5, -2.5, -2.5, -2.5, -2.5>
                var right = Vector256.Create(5.0f);                      // <5, 5, 5, 5, 5, 5, 5, 5>
                Vector256 <float> result = Avx.AddSubtract(left, right); // result = <-7.5, 2.5, -7.5, 2.5, -7.5, 2.5, -7.5, 2.5>xit
                left   = Vector256.Create(-1.0f, -2.0f, -3.0f, -4.0f, -50.0f, -60.0f, -70.0f, -80.0f);
                right  = Vector256.Create(0.0f, 2.0f, 3.0f, 4.0f, 50.0f, 60.0f, 70.0f, 80.0f);
                result = Avx.UnpackHigh(left, right);              // result = <-3, 3, -4, 4, -70, 70, -80, 80>
                result = Avx.UnpackLow(left, right);               // result = <-1, 1, -2, 2, -50, 50, -60, 60>
                result = Avx.DotProduct(left, right, 0b1111_0001); // result = <-30, 0, 0, 0, -17400, 0, 0, 0>
                bool testResult = Avx.TestC(left, right);          // testResult = true
                testResult = Avx.TestC(right, left);               // testResult = false
                Vector256 <float> result1 = Avx.Divide(left, right);
                var plusOne = Vector256.Create(1.0f);
                result = Avx.Compare(right, result1, FloatComparisonMode.OrderedGreaterThanNonSignaling);
                result = Avx.Compare(right, result1, FloatComparisonMode.UnorderedNotLessThanNonSignaling);
                left   = Vector256.Create(0.0f, 3.0f, -3.0f, 4.0f, -50.0f, 60.0f, -70.0f, 80.0f);
                right  = Vector256.Create(0.0f, 2.0f, 3.0f, 2.0f, 50.0f, -60.0f, 70.0f, -80.0f);
                Vector256 <float> nanInFirstPosition = Avx.Divide(left, right);
                left = Vector256.Create(1.1f, 3.3333333f, -3.0f, 4.22f, -50.0f, 60.0f, -70.0f, 80.0f);
                Vector256 <float> InfInFirstPosition = Avx.Divide(left, right);

                left  = Vector256.Create(-1.1f, 3.0f, 1.0f / 3.0f, MathF.PI, -50.0f, 60.0f, -70.0f, 80.0f);
                right = Vector256.Create(0.0f, 2.0f, 3.1f, 2.0f, 50.0f, -60.0f, 70.0f, -80.0f);
                Vector256 <float> compareResult = Avx.Compare(left, right, FloatComparisonMode.OrderedGreaterThanNonSignaling); // compareResult = <0, NaN, 0, NaN, 0, NaN, 0, NaN>
                Vector256 <float> mixed         = Avx.BlendVariable(left, right, compareResult);                                //  mixed = <-1, 2, -3, 2, -50, -60, -70, -80>

                //left = Vector256.Create(-1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f);
                //right = Vector256.Create(1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f);
                Vector256 <float> other = right = Vector256.Create(0.0f, 2.0f, 3.0f, 2.0f, 50.0f, -60.0f, 70.0f, -80.0f);
                bool bRes    = Avx.TestZ(plusOne, compareResult);
                bool bRes2   = Avx.TestC(plusOne, compareResult);
                bool allTrue = !Avx.TestZ(compareResult, compareResult);
                compareResult = Avx.Compare(nanInFirstPosition, right, FloatComparisonMode.OrderedEqualNonSignaling); // compareResult = <0, NaN, 0, NaN, 0, NaN, 0, NaN>
                compareResult = Avx.Compare(nanInFirstPosition, right, FloatComparisonMode.UnorderedEqualNonSignaling);
                compareResult = Avx.Compare(InfInFirstPosition, right, FloatComparisonMode.UnorderedNotLessThanOrEqualNonSignaling);
                compareResult = Avx.Compare(InfInFirstPosition, right, FloatComparisonMode.OrderedGreaterThanNonSignaling);
                var left128  = Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f);
                var right128 = Vector128.Create(2.0f, 3.0f, 4.0f, 5.0f);
                Vector128 <float> compResult128 = Sse.CompareGreaterThan(left128, right128); // compResult128 = <0, 0, 0, 0>

                int res = Avx.MoveMask(compareResult);
                if (Fma.IsSupported)
                {
                    Vector256 <float> resultFma = Fma.MultiplyAdd(left, right, other); // = left * right + other for each element
                    resultFma = Fma.MultiplyAddNegated(left, right, other);            // = -(left * right + other) for each element
                    resultFma = Fma.MultiplySubtract(left, right, other);              // = left * right - other for each element
                    Fma.MultiplyAddSubtract(left, right, other);                       // even elements (0, 2, ...) like MultiplyAdd, odd elements like MultiplySubtract
                }
                result = Avx.DotProduct(left, right, 0b1010_0001);                     // result = <-20, 0, 0, 0, -10000, 0, 0, 0>
                result = Avx.Floor(left);                                              // result = <-3, -3, -3, -3, -3, -3, -3, -3>
                result = Avx.Add(left, right);                                         // result = <2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5>
                result = Avx.Ceiling(left);                                            // result = <-2, -2, -2, -2, -2, -2, -2, -2>
                result = Avx.Multiply(left, right);                                    // result = <-12.5, -12.5, -12.5, -12.5, -12.5, -12.5, -12.5, -12.5>
                result = Avx.HorizontalAdd(left, right);                               // result = <-5, -5, 10, 10, -5, -5, 10, 10>
                result = Avx.HorizontalSubtract(left, right);                          // result = <0, 0, 0, 0, 0, 0, 0, 0>
                double[] someDoubles      = new double[] { 1.0, 3.0, -2.5, 7.5, 10.8, 0.33333 };
                double[] someOtherDoubles = new double[] { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 };
                double[] someResult       = new double[someDoubles.Length];
                float[]  someFloats       = new float[] { 1, 2, 3, 4, 10, 20, 30, 40, 0 };
                float[]  someOtherFloats  = new float[] { 1, 1, 1, 1, 1, 1, 1, 1 };
                unsafe
                {
                    fixed(double *ptr = &someDoubles[1])
                    {
                        fixed(double *ptr2 = &someResult[0])
                        {
                            Vector256 <double> res2 = Avx.LoadVector256(ptr); // res2 = <3, -2.5, 7.5, 10.8>

                            Avx.Store(ptr2, res2);
                        }
                    }

                    fixed(float *ptr = &someFloats[0])
                    {
                        fixed(float *ptr2 = &someOtherFloats[0])
                        {
                            Vector256 <float> res2 = Avx.DotProduct(Avx.LoadVector256(ptr), Avx.LoadVector256(ptr2), 0b0001_0001);
                            //Avx.Store(ptr2, res2);
                        }
                    }
                }
            }
        }