private uint Hash(ref double value, uint seed)
        {
            ulong v    = FloatUtils.GetBits(value);
            uint  hash = Hashing.MurmurRound(seed, Utils.GetLo(v));

            return(Hashing.MurmurRound(hash, Utils.GetHi(v)));
        }
示例#2
0
        public void SavePipeKeyToVec()
        {
            string pathTerms = DeleteOutputPath("SavePipe", "Terms.txt");

            File.WriteAllLines(pathTerms, new string[] {
                "Black",
                "White",
                "Male",
                "Female"
            });

            string pathData = GetDataPath("adult.test");

            TestCore(pathData, true,
                     new[] {
                "loader=Text{header+ sep=comma col=Mar:TX:5 col=Race:TX:8 col=Gen:TX:8~9}",
                "xf=Concat{col=Comb:Race,Gen,Race}",
                "xf=Cat{kind=Key col=MarKey:Mar}",
                "xf=Cat{kind=Key col={name=CombKey src=Comb} data={" + pathTerms + "}}",
                "xf=Convert{col=MarKeyU8:U8:MarKey col=CombKeyU1:U1:CombKey}",
                "xf=KeyToVector{col={name=CombBagVec src=CombKey bag+} col={name=CombIndVec src=CombKey} col=MarVec:MarKey}",
                "xf=KeyToVector{col={name=CombBagVecU1 src=CombKeyU1 bag+} col={name=CombIndVecU1 src=CombKeyU1} col=MarVecU8:MarKeyU8}",
                "xf=ChooseColumns{col=MarKey col=CombKey col=MarVec col=MarVecU8 col=CombBagVec col=CombBagVecU1 col=CombIndVec col=CombIndVecU1 col=Mar col=Comb}",
            },

                     pipe =>
            {
                // Verify that the Vec columns match the corresponding VecXX columns. This verifies that conversion
                // happened correctly in KeyToVector.
                using (var c = pipe.GetRowCursor(col => true))
                {
                    var cols    = new[] { "MarVec", "MarVecU8", "CombBagVec", "CombBagVecU1", "CombIndVec", "CombIndVecU1" };
                    var getters = new ValueGetter <VBuffer <Float> > [cols.Length];
                    for (int i = 0; i < cols.Length; i++)
                    {
                        int col;
                        if (!Check(c.Schema.TryGetColumnIndex(cols[i], out col), "{0} not found!", cols[i]))
                        {
                            return;
                        }
                        getters[i] = c.GetGetter <VBuffer <Float> >(col);
                    }

                    Func <Float, Float, bool> fn = (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y);
                    var v1 = default(VBuffer <Float>);
                    var v2 = default(VBuffer <Float>);
                    while (c.MoveNext())
                    {
                        for (int i = 0; i < cols.Length; i += 2)
                        {
                            getters[i](ref v1);
                            getters[i + 1](ref v2);
                            Check(CompareVec(ref v1, ref v2, v1.Length, fn), "Mismatch");
                        }
                    }
                }
            });

            Done();
        }
 private uint Hash(ref float value, uint seed)
 {
     return(Hashing.MurmurRound(seed, FloatUtils.GetBits(value)));
 }
 private static bool EqualWithEps(Double x, Double y)
 {
     // bitwise comparison is needed because Abs(Inf-Inf) and Abs(NaN-NaN) are not 0s.
     return(FloatUtils.GetBits(x) == FloatUtils.GetBits(y) || Math.Abs(x - y) < DoubleEps);
 }
示例#5
0
        protected Func <bool> GetColumnComparer(DataViewRow r1, DataViewRow r2, int col, DataViewType type, bool exactDoubles)
        {
            if (type is VectorDataViewType vecType)
            {
                int size = vecType.Size;
                Contracts.Assert(size >= 0);
                var result = vecType.ItemType.RawType.TryGetDataKind(out var kind);
                Contracts.Assert(result);

                switch (kind)
                {
                case InternalDataKind.I1:
                    return(GetComparerVec <sbyte>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.U1:
                    return(GetComparerVec <byte>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.I2:
                    return(GetComparerVec <short>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.U2:
                    return(GetComparerVec <ushort>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.I4:
                    return(GetComparerVec <int>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.U4:
                    return(GetComparerVec <uint>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.I8:
                    return(GetComparerVec <long>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.U8:
                    return(GetComparerVec <ulong>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.R4:
                    return(GetComparerVec <Single>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));

                case InternalDataKind.R8:
                    if (exactDoubles)
                    {
                        return(GetComparerVec <Double>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
                    }
                    else
                    {
                        return(GetComparerVec <Double>(r1, r2, col, size, EqualWithEps));
                    }

                case InternalDataKind.Text:
                    return(GetComparerVec <ReadOnlyMemory <char> >(r1, r2, col, size, (a, b) => a.Span.SequenceEqual(b.Span)));

                case InternalDataKind.Bool:
                    return(GetComparerVec <bool>(r1, r2, col, size, (x, y) => x == y));

                case InternalDataKind.TimeSpan:
                    return(GetComparerVec <TimeSpan>(r1, r2, col, size, (x, y) => x.Ticks == y.Ticks));

                case InternalDataKind.DT:
                    return(GetComparerVec <DateTime>(r1, r2, col, size, (x, y) => x.Ticks == y.Ticks));

                case InternalDataKind.DZ:
                    return(GetComparerVec <DateTimeOffset>(r1, r2, col, size, (x, y) => x.Equals(y)));

                case InternalDataKind.UG:
                    return(GetComparerVec <DataViewRowId>(r1, r2, col, size, (x, y) => x.Equals(y)));
                }
            }
            else
            {
                var result = type.RawType.TryGetDataKind(out var kind);
                Contracts.Assert(result);
                switch (kind)
                {
                case InternalDataKind.I1:
                    return(GetComparerOne <sbyte>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.U1:
                    return(GetComparerOne <byte>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.I2:
                    return(GetComparerOne <short>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.U2:
                    return(GetComparerOne <ushort>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.I4:
                    return(GetComparerOne <int>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.U4:
                    return(GetComparerOne <uint>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.I8:
                    return(GetComparerOne <long>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.U8:
                    return(GetComparerOne <ulong>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.R4:
                    return(GetComparerOne <Single>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));

                case InternalDataKind.R8:
                    if (exactDoubles)
                    {
                        return(GetComparerOne <Double>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
                    }
                    else
                    {
                        return(GetComparerOne <Double>(r1, r2, col, EqualWithEps));
                    }

                case InternalDataKind.Text:
                    return(GetComparerOne <ReadOnlyMemory <char> >(r1, r2, col, (a, b) => a.Span.SequenceEqual(b.Span)));

                case InternalDataKind.Bool:
                    return(GetComparerOne <bool>(r1, r2, col, (x, y) => x == y));

                case InternalDataKind.TimeSpan:
                    return(GetComparerOne <TimeSpan>(r1, r2, col, (x, y) => x.Ticks == y.Ticks));

                case InternalDataKind.DT:
                    return(GetComparerOne <DateTime>(r1, r2, col, (x, y) => x.Ticks == y.Ticks));

                case InternalDataKind.DZ:
                    return(GetComparerOne <DateTimeOffset>(r1, r2, col, (x, y) => x.Equals(y)));

                case InternalDataKind.UG:
                    return(GetComparerOne <DataViewRowId>(r1, r2, col, (x, y) => x.Equals(y)));
                }
            }

#if !CORECLR // REVIEW: Port Picture type to CoreTLC.
            if (type is PictureType)
            {
                var     g1 = r1.GetGetter <Picture>(col);
                var     g2 = r2.GetGetter <Picture>(col);
                Picture v1 = null;
                Picture v2 = null;
                return
                    (() =>
                {
                    g1(ref v1);
                    g2(ref v2);
                    return ComparePicture(v1, v2);
                });
            }
#endif

            throw Contracts.Except("Unknown type in GetColumnComparer: '{0}'", type);
        }
        protected Func <bool> GetColumnComparer(IRow r1, IRow r2, int col, ColumnType type, bool exactDoubles)
        {
            if (!type.IsVector)
            {
                switch (type.RawKind)
                {
                case DataKind.I1:
                    return(GetComparerOne <DvInt1>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U1:
                    return(GetComparerOne <byte>(r1, r2, col, (x, y) => x == y));

                case DataKind.I2:
                    return(GetComparerOne <DvInt2>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U2:
                    return(GetComparerOne <ushort>(r1, r2, col, (x, y) => x == y));

                case DataKind.I4:
                    return(GetComparerOne <DvInt4>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U4:
                    return(GetComparerOne <uint>(r1, r2, col, (x, y) => x == y));

                case DataKind.I8:
                    return(GetComparerOne <DvInt8>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U8:
                    return(GetComparerOne <ulong>(r1, r2, col, (x, y) => x == y));

                case DataKind.R4:
                    return(GetComparerOne <Single>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));

                case DataKind.R8:
                    if (exactDoubles)
                    {
                        return(GetComparerOne <Double>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
                    }
                    else
                    {
                        return(GetComparerOne <Double>(r1, r2, col, EqualWithEps));
                    }

                case DataKind.Text:
                    return(GetComparerOne <DvText>(r1, r2, col, DvText.Identical));

                case DataKind.Bool:
                    return(GetComparerOne <DvBool>(r1, r2, col, (x, y) => x.Equals(y)));

                case DataKind.TimeSpan:
                    return(GetComparerOne <DvTimeSpan>(r1, r2, col, (x, y) => x.Equals(y)));

                case DataKind.DT:
                    return(GetComparerOne <DvDateTime>(r1, r2, col, (x, y) => x.Equals(y)));

                case DataKind.DZ:
                    return(GetComparerOne <DvDateTimeZone>(r1, r2, col, (x, y) => x.Equals(y)));

                case DataKind.UG:
                    return(GetComparerOne <UInt128>(r1, r2, col, (x, y) => x.Equals(y)));
                }
            }
            else
            {
                int size = type.VectorSize;
                Contracts.Assert(size >= 0);
                switch (type.ItemType.RawKind)
                {
                case DataKind.I1:
                    return(GetComparerVec <DvInt1>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U1:
                    return(GetComparerVec <byte>(r1, r2, col, size, (x, y) => x == y));

                case DataKind.I2:
                    return(GetComparerVec <DvInt2>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U2:
                    return(GetComparerVec <ushort>(r1, r2, col, size, (x, y) => x == y));

                case DataKind.I4:
                    return(GetComparerVec <DvInt4>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U4:
                    return(GetComparerVec <uint>(r1, r2, col, size, (x, y) => x == y));

                case DataKind.I8:
                    return(GetComparerVec <DvInt8>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));

                case DataKind.U8:
                    return(GetComparerVec <ulong>(r1, r2, col, size, (x, y) => x == y));

                case DataKind.R4:
                    return(GetComparerVec <Single>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));

                case DataKind.R8:
                    if (exactDoubles)
                    {
                        return(GetComparerVec <Double>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
                    }
                    else
                    {
                        return(GetComparerVec <Double>(r1, r2, col, size, EqualWithEps));
                    }

                case DataKind.Text:
                    return(GetComparerVec <DvText>(r1, r2, col, size, DvText.Identical));

                case DataKind.Bool:
                    return(GetComparerVec <DvBool>(r1, r2, col, size, (x, y) => x.Equals(y)));

                case DataKind.TimeSpan:
                    return(GetComparerVec <DvTimeSpan>(r1, r2, col, size, (x, y) => x.Equals(y)));

                case DataKind.DT:
                    return(GetComparerVec <DvDateTime>(r1, r2, col, size, (x, y) => x.Equals(y)));

                case DataKind.DZ:
                    return(GetComparerVec <DvDateTimeZone>(r1, r2, col, size, (x, y) => x.Equals(y)));

                case DataKind.UG:
                    return(GetComparerVec <UInt128>(r1, r2, col, size, (x, y) => x.Equals(y)));
                }
            }

#if !CORECLR // REVIEW: Port Picture type to CoreTLC.
            if (type is PictureType)
            {
                var     g1 = r1.GetGetter <Picture>(col);
                var     g2 = r2.GetGetter <Picture>(col);
                Picture v1 = null;
                Picture v2 = null;
                return
                    (() =>
                {
                    g1(ref v1);
                    g2(ref v2);
                    return ComparePicture(v1, v2);
                });
            }
#endif

            throw Contracts.Except("Unknown type in GetColumnComparer: '{0}'", type);
        }