private uint Hash(ref double value, uint seed)
{
ulong v = FloatUtils.GetBits(value);
uint hash = Hashing.MurmurRound(seed, Utils.GetLo(v));
return(Hashing.MurmurRound(hash, Utils.GetHi(v)));
}
public void SavePipeKeyToVec()
{
string pathTerms = DeleteOutputPath("SavePipe", "Terms.txt");
File.WriteAllLines(pathTerms, new string[] {
"Black",
"White",
"Male",
"Female"
});
string pathData = GetDataPath("adult.test");
TestCore(pathData, true,
new[] {
"loader=Text{header+ sep=comma col=Mar:TX:5 col=Race:TX:8 col=Gen:TX:8~9}",
"xf=Concat{col=Comb:Race,Gen,Race}",
"xf=Cat{kind=Key col=MarKey:Mar}",
"xf=Cat{kind=Key col={name=CombKey src=Comb} data={" + pathTerms + "}}",
"xf=Convert{col=MarKeyU8:U8:MarKey col=CombKeyU1:U1:CombKey}",
"xf=KeyToVector{col={name=CombBagVec src=CombKey bag+} col={name=CombIndVec src=CombKey} col=MarVec:MarKey}",
"xf=KeyToVector{col={name=CombBagVecU1 src=CombKeyU1 bag+} col={name=CombIndVecU1 src=CombKeyU1} col=MarVecU8:MarKeyU8}",
"xf=ChooseColumns{col=MarKey col=CombKey col=MarVec col=MarVecU8 col=CombBagVec col=CombBagVecU1 col=CombIndVec col=CombIndVecU1 col=Mar col=Comb}",
},
pipe =>
{
// Verify that the Vec columns match the corresponding VecXX columns. This verifies that conversion
// happened correctly in KeyToVector.
using (var c = pipe.GetRowCursor(col => true))
{
var cols = new[] { "MarVec", "MarVecU8", "CombBagVec", "CombBagVecU1", "CombIndVec", "CombIndVecU1" };
var getters = new ValueGetter <VBuffer <Float> > [cols.Length];
for (int i = 0; i < cols.Length; i++)
{
int col;
if (!Check(c.Schema.TryGetColumnIndex(cols[i], out col), "{0} not found!", cols[i]))
{
return;
}
getters[i] = c.GetGetter <VBuffer <Float> >(col);
}
Func <Float, Float, bool> fn = (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y);
var v1 = default(VBuffer <Float>);
var v2 = default(VBuffer <Float>);
while (c.MoveNext())
{
for (int i = 0; i < cols.Length; i += 2)
{
getters[i](ref v1);
getters[i + 1](ref v2);
Check(CompareVec(ref v1, ref v2, v1.Length, fn), "Mismatch");
}
}
}
});
Done();
}
private uint Hash(ref float value, uint seed)
{
return(Hashing.MurmurRound(seed, FloatUtils.GetBits(value)));
}
private static bool EqualWithEps(Double x, Double y)
{
// bitwise comparison is needed because Abs(Inf-Inf) and Abs(NaN-NaN) are not 0s.
return(FloatUtils.GetBits(x) == FloatUtils.GetBits(y) || Math.Abs(x - y) < DoubleEps);
}
protected Func <bool> GetColumnComparer(DataViewRow r1, DataViewRow r2, int col, DataViewType type, bool exactDoubles)
{
if (type is VectorDataViewType vecType)
{
int size = vecType.Size;
Contracts.Assert(size >= 0);
var result = vecType.ItemType.RawType.TryGetDataKind(out var kind);
Contracts.Assert(result);
switch (kind)
{
case InternalDataKind.I1:
return(GetComparerVec <sbyte>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.U1:
return(GetComparerVec <byte>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.I2:
return(GetComparerVec <short>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.U2:
return(GetComparerVec <ushort>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.I4:
return(GetComparerVec <int>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.U4:
return(GetComparerVec <uint>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.I8:
return(GetComparerVec <long>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.U8:
return(GetComparerVec <ulong>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.R4:
return(GetComparerVec <Single>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
case InternalDataKind.R8:
if (exactDoubles)
{
return(GetComparerVec <Double>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
}
else
{
return(GetComparerVec <Double>(r1, r2, col, size, EqualWithEps));
}
case InternalDataKind.Text:
return(GetComparerVec <ReadOnlyMemory <char> >(r1, r2, col, size, (a, b) => a.Span.SequenceEqual(b.Span)));
case InternalDataKind.Bool:
return(GetComparerVec <bool>(r1, r2, col, size, (x, y) => x == y));
case InternalDataKind.TimeSpan:
return(GetComparerVec <TimeSpan>(r1, r2, col, size, (x, y) => x.Ticks == y.Ticks));
case InternalDataKind.DT:
return(GetComparerVec <DateTime>(r1, r2, col, size, (x, y) => x.Ticks == y.Ticks));
case InternalDataKind.DZ:
return(GetComparerVec <DateTimeOffset>(r1, r2, col, size, (x, y) => x.Equals(y)));
case InternalDataKind.UG:
return(GetComparerVec <DataViewRowId>(r1, r2, col, size, (x, y) => x.Equals(y)));
}
}
else
{
var result = type.RawType.TryGetDataKind(out var kind);
Contracts.Assert(result);
switch (kind)
{
case InternalDataKind.I1:
return(GetComparerOne <sbyte>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.U1:
return(GetComparerOne <byte>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.I2:
return(GetComparerOne <short>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.U2:
return(GetComparerOne <ushort>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.I4:
return(GetComparerOne <int>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.U4:
return(GetComparerOne <uint>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.I8:
return(GetComparerOne <long>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.U8:
return(GetComparerOne <ulong>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.R4:
return(GetComparerOne <Single>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
case InternalDataKind.R8:
if (exactDoubles)
{
return(GetComparerOne <Double>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
}
else
{
return(GetComparerOne <Double>(r1, r2, col, EqualWithEps));
}
case InternalDataKind.Text:
return(GetComparerOne <ReadOnlyMemory <char> >(r1, r2, col, (a, b) => a.Span.SequenceEqual(b.Span)));
case InternalDataKind.Bool:
return(GetComparerOne <bool>(r1, r2, col, (x, y) => x == y));
case InternalDataKind.TimeSpan:
return(GetComparerOne <TimeSpan>(r1, r2, col, (x, y) => x.Ticks == y.Ticks));
case InternalDataKind.DT:
return(GetComparerOne <DateTime>(r1, r2, col, (x, y) => x.Ticks == y.Ticks));
case InternalDataKind.DZ:
return(GetComparerOne <DateTimeOffset>(r1, r2, col, (x, y) => x.Equals(y)));
case InternalDataKind.UG:
return(GetComparerOne <DataViewRowId>(r1, r2, col, (x, y) => x.Equals(y)));
}
}
#if !CORECLR // REVIEW: Port Picture type to CoreTLC.
if (type is PictureType)
{
var g1 = r1.GetGetter <Picture>(col);
var g2 = r2.GetGetter <Picture>(col);
Picture v1 = null;
Picture v2 = null;
return
(() =>
{
g1(ref v1);
g2(ref v2);
return ComparePicture(v1, v2);
});
}
#endif
throw Contracts.Except("Unknown type in GetColumnComparer: '{0}'", type);
}
protected Func <bool> GetColumnComparer(IRow r1, IRow r2, int col, ColumnType type, bool exactDoubles)
{
if (!type.IsVector)
{
switch (type.RawKind)
{
case DataKind.I1:
return(GetComparerOne <DvInt1>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));
case DataKind.U1:
return(GetComparerOne <byte>(r1, r2, col, (x, y) => x == y));
case DataKind.I2:
return(GetComparerOne <DvInt2>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));
case DataKind.U2:
return(GetComparerOne <ushort>(r1, r2, col, (x, y) => x == y));
case DataKind.I4:
return(GetComparerOne <DvInt4>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));
case DataKind.U4:
return(GetComparerOne <uint>(r1, r2, col, (x, y) => x == y));
case DataKind.I8:
return(GetComparerOne <DvInt8>(r1, r2, col, (x, y) => x.RawValue == y.RawValue));
case DataKind.U8:
return(GetComparerOne <ulong>(r1, r2, col, (x, y) => x == y));
case DataKind.R4:
return(GetComparerOne <Single>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
case DataKind.R8:
if (exactDoubles)
{
return(GetComparerOne <Double>(r1, r2, col, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
}
else
{
return(GetComparerOne <Double>(r1, r2, col, EqualWithEps));
}
case DataKind.Text:
return(GetComparerOne <DvText>(r1, r2, col, DvText.Identical));
case DataKind.Bool:
return(GetComparerOne <DvBool>(r1, r2, col, (x, y) => x.Equals(y)));
case DataKind.TimeSpan:
return(GetComparerOne <DvTimeSpan>(r1, r2, col, (x, y) => x.Equals(y)));
case DataKind.DT:
return(GetComparerOne <DvDateTime>(r1, r2, col, (x, y) => x.Equals(y)));
case DataKind.DZ:
return(GetComparerOne <DvDateTimeZone>(r1, r2, col, (x, y) => x.Equals(y)));
case DataKind.UG:
return(GetComparerOne <UInt128>(r1, r2, col, (x, y) => x.Equals(y)));
}
}
else
{
int size = type.VectorSize;
Contracts.Assert(size >= 0);
switch (type.ItemType.RawKind)
{
case DataKind.I1:
return(GetComparerVec <DvInt1>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));
case DataKind.U1:
return(GetComparerVec <byte>(r1, r2, col, size, (x, y) => x == y));
case DataKind.I2:
return(GetComparerVec <DvInt2>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));
case DataKind.U2:
return(GetComparerVec <ushort>(r1, r2, col, size, (x, y) => x == y));
case DataKind.I4:
return(GetComparerVec <DvInt4>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));
case DataKind.U4:
return(GetComparerVec <uint>(r1, r2, col, size, (x, y) => x == y));
case DataKind.I8:
return(GetComparerVec <DvInt8>(r1, r2, col, size, (x, y) => x.RawValue == y.RawValue));
case DataKind.U8:
return(GetComparerVec <ulong>(r1, r2, col, size, (x, y) => x == y));
case DataKind.R4:
return(GetComparerVec <Single>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
case DataKind.R8:
if (exactDoubles)
{
return(GetComparerVec <Double>(r1, r2, col, size, (x, y) => FloatUtils.GetBits(x) == FloatUtils.GetBits(y)));
}
else
{
return(GetComparerVec <Double>(r1, r2, col, size, EqualWithEps));
}
case DataKind.Text:
return(GetComparerVec <DvText>(r1, r2, col, size, DvText.Identical));
case DataKind.Bool:
return(GetComparerVec <DvBool>(r1, r2, col, size, (x, y) => x.Equals(y)));
case DataKind.TimeSpan:
return(GetComparerVec <DvTimeSpan>(r1, r2, col, size, (x, y) => x.Equals(y)));
case DataKind.DT:
return(GetComparerVec <DvDateTime>(r1, r2, col, size, (x, y) => x.Equals(y)));
case DataKind.DZ:
return(GetComparerVec <DvDateTimeZone>(r1, r2, col, size, (x, y) => x.Equals(y)));
case DataKind.UG:
return(GetComparerVec <UInt128>(r1, r2, col, size, (x, y) => x.Equals(y)));
}
}
#if !CORECLR // REVIEW: Port Picture type to CoreTLC.
if (type is PictureType)
{
var g1 = r1.GetGetter <Picture>(col);
var g2 = r2.GetGetter <Picture>(col);
Picture v1 = null;
Picture v2 = null;
return
(() =>
{
g1(ref v1);
g2(ref v2);
return ComparePicture(v1, v2);
});
}
#endif
throw Contracts.Except("Unknown type in GetColumnComparer: '{0}'", type);
}