public int FilterLandRegistryRecordsArrowVectorized2()
{
var recordCount = recordBatch.Length;
//var selectMask = new byte[recordCount]; // new int[0];
int[] selectMask = new int[recordCount];
const long MillisecondsPerDay = 86400000;
var dateFilter = (int)(DateTimeOffset.Parse("2019-01-01").ToUnixTimeMilliseconds() / MillisecondsPerDay);
var dateValues = (recordBatch.Column(0) as Date32Array).Values;
VectorizedFiltering.ApplyFilterMask <int, int>(
dateValues, Vector.GreaterThanOrEqual, (v, f) => v >= f, dateFilter,
selectMask, Vector.BitwiseOr, (f1, f2) => (byte)(f1 | f2));
var priceValues = (recordBatch.Column(1) as FloatArray).Values;
VectorizedFiltering.ApplyFilterMask <float, int>(
priceValues, Vector.GreaterThan, (v, f) => v > f, 5000000,
selectMask, Vector.BitwiseAnd, (f1, f2) => f1 & f2);
byte[] byteSelectMask = VectorizedFiltering.NarrowFilterMask(selectMask);
var stringEncoding = Encoding.ASCII;
var propertyTypeFilter = new string[] { "D", "S", "T" }.Select(x => stringEncoding.GetBytes(x)[0]).ToArray();
var propertyTypeValues = (recordBatch.Column(2) as StringArray).Values;
byte[] byteSelectMask2 = VectorizedFiltering.GetFilterMask <byte>(
propertyTypeValues, Vector.Equals, (v, f) => v == f, propertyTypeFilter);
byteSelectMask = VectorizedFiltering.CombineFilterMask(
byteSelectMask, byteSelectMask2, Vector.BitwiseAnd, (f1, f2) => (byte)(f1 & f2));
var tenureFilter = stringEncoding.GetBytes("F")[0];
var tenureValues = (recordBatch.Column(3) as StringArray).Values;
VectorizedFiltering.ApplyFilterMask <byte, byte>(
tenureValues, Vector.Equals, (v, f) => v == f, tenureFilter,
byteSelectMask, Vector.BitwiseAnd, (f1, f2) => (byte)(f1 & f2));
var count = VectorizedFiltering.CountFilterMask(byteSelectMask);
#if LOG
Console.WriteLine("Found {0} records", count);
#endif
return(count);
}
public unsafe int FilterLandRegistryRecordsArrowVectorizedAvx()
{
var recordCount = recordBatch.Length;
var selectMask = new int[recordCount]; // new int[0];
const long MillisecondsPerDay = 86400000;
var dateFilter = (int)(DateTimeOffset.Parse("2019-01-01").ToUnixTimeMilliseconds() / MillisecondsPerDay);
var dateFilterVector = Vector256.Create(dateFilter);
var dateFilterVectorLimit = recordCount - (recordCount % Vector256 <int> .Count);
var dateValues = (recordBatch.Column(0) as Date32Array).Values;
fixed(int *pDateValues = dateValues, pSelectMask = selectMask)
{
for (var i = 0; i < dateFilterVectorLimit; i += Vector256 <int> .Count)
{
var resultVector = Avx2.CompareGreaterThan(
Avx2.LoadVector256(pDateValues + i),
dateFilterVector
);
Avx2.Store(pSelectMask + i, resultVector);
}
}
for (var i = dateFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = dateValues[i] >= dateFilter;
selectMask[i] = Unsafe.As <bool, int>(ref predicateResult);
//selectMask[i] = dateValues[i] >= dateFilter ? -1 : 0;
}
var priceFilterVector = Vector256.Create(5000000f).AsInt32();
var priceFilterVectorLimit = recordCount - (recordCount % Vector256 <float> .Count);
var priceValues = (recordBatch.Column(1) as FloatArray).Values;
fixed(float *pPriceValues = priceValues)
fixed(int *pSelectMask = selectMask)
{
for (var i = 0; i < priceFilterVectorLimit; i += Vector256 <float> .Count)
{
var resultVector = Avx2.And(
Avx2.LoadVector256(pSelectMask + i),
Avx2.CompareGreaterThan(
Avx2.LoadVector256(pPriceValues + i).AsInt32(),
priceFilterVector
)
);
Avx2.Store(pSelectMask + i, resultVector);
}
}
for (var i = priceFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = priceValues[i] > 5000000;
selectMask[i] &= Unsafe.As <bool, int>(ref predicateResult);
//selectMask[i] = selectMask[i] && priceValues[i] > 5000000 ? -1 : 0;
}
byte[] byteSelectMask = VectorizedFiltering.NarrowFilterMask(selectMask);
var stringEncoding = Encoding.ASCII;
var propertyTypeSelectMask = new byte[byteSelectMask.Length];
var propertyTypeFilterList = new string[] { "D", "S", "T" }.Select(x => stringEncoding.GetBytes(x)[0]).ToArray();
var propertyTypeFilterVectorLimit = recordCount - (recordCount % Vector256 <byte> .Count);
for (var j = 0; j < propertyTypeFilterList.Length; j++)
{
var propertyTypeFilter = propertyTypeFilterList[j];
var propertyTypeFilterVector = Vector256.Create(propertyTypeFilter).AsSByte();
var propertyTypeValues = (recordBatch.Column(2) as StringArray).Values;
fixed(byte *pPropertyTypeValues = propertyTypeValues, pPropertyTypeSelectMask = propertyTypeSelectMask)
{
for (var i = 0; i < propertyTypeFilterVectorLimit; i += Vector256 <byte> .Count)
{
var resultVector = Avx2.And(
Avx2.LoadVector256(pPropertyTypeSelectMask + i),
Avx2.CompareGreaterThan(
Avx2.LoadVector256(pPropertyTypeValues + i).AsSByte(),
propertyTypeFilterVector
).AsByte()
);
Avx2.Store(pPropertyTypeSelectMask + i, resultVector);
}
}
for (var i = propertyTypeFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = propertyTypeValues[i] == propertyTypeFilter;
propertyTypeSelectMask[i] |= Unsafe.As <bool, byte>(ref predicateResult);
//byteSelectMask[i] = tenureValues[i] == tenureFilter ? -1 : 0;
}
//for (var i = 0; i < recordCount; i++)
//{
// selectMask[i] = selectMask[i] && propertyTypeFilter.Contains(propertyTypeValues[i]);
//}
}
fixed(byte *pByteSelectMask = byteSelectMask, pPropertyTypeSelectMask = propertyTypeSelectMask)
{
for (var i = 0; i < propertyTypeFilterVectorLimit; i += Vector256 <byte> .Count)
{
var resultVector = Avx2.And(
Avx2.LoadVector256(pByteSelectMask + i),
Avx2.LoadVector256(pPropertyTypeSelectMask + i)
);
Avx2.Store(pByteSelectMask + i, resultVector);
}
}
for (var i = propertyTypeFilterVectorLimit; i < recordCount; i++)
{
byteSelectMask[i] &= propertyTypeSelectMask[i];
}
var tenureFilter = stringEncoding.GetBytes("F")[0];
var tenureFilterVector = Vector256.Create(tenureFilter).AsSByte();
var tenureFilterVectorLimit = recordCount - (recordCount % Vector256 <byte> .Count);
var tenureValues = (recordBatch.Column(3) as StringArray).Values;
fixed(byte *pByteSelectMask = byteSelectMask, pTenureValues = tenureValues)
{
for (var i = 0; i < tenureFilterVectorLimit; i += Vector256 <byte> .Count)
{
var resultVector = Avx2.And(
Avx2.LoadVector256(pByteSelectMask + i),
Avx2.CompareGreaterThan(
Avx2.LoadVector256(pTenureValues + i).AsSByte(),
tenureFilterVector
).AsByte()
);
Avx2.Store(pByteSelectMask + i, resultVector);
}
}
for (var i = tenureFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = tenureValues[i] == tenureFilter;
byteSelectMask[i] &= Unsafe.As <bool, byte>(ref predicateResult);
//byteSelectMask[i] = byteSelectMask[i] && tenureValues[i] == tenureFilter ? -1 : 0;
}
var countVector = Vector256 <byte> .Zero;
var oneVector = Vector256.Create((byte)1);
fixed(byte *pByteSelectMask = byteSelectMask, pTenureValues = tenureValues)
{
for (var i = 0; i < tenureFilterVectorLimit; i += Vector256 <byte> .Count)
{
countVector = Avx2.Add(
countVector,
Avx2.And(Avx2.LoadVector256(pByteSelectMask + i), oneVector)
);
}
}
var count = 0;
for (var i = 0; i < Vector256 <byte> .Count; i++)
{
count += countVector.GetElement(i);
}
for (var i = tenureFilterVectorLimit; i < recordCount; i++)
{
count += byteSelectMask[i] & 1;
}
#if LOG
Console.WriteLine("Found {0} records", count);
#endif
return(count);
//return selectMask.Count(v => v == -1);
}
public int FilterLandRegistryRecordsArrowVectorized()
{
var recordCount = recordBatch.Length;
var selectMask = new int[recordCount]; // new int[0];
const long MillisecondsPerDay = 86400000;
var dateFilter = (int)(DateTimeOffset.Parse("2019-01-01").ToUnixTimeMilliseconds() / MillisecondsPerDay);
var dateFilterVector = new Vector <int>(dateFilter);
var dateFilterVectorLimit = recordCount - (recordCount % Vector <int> .Count);
var dateValues = (recordBatch.Column(0) as Date32Array).Values;
for (var i = 0; i < dateFilterVectorLimit; i += Vector <int> .Count)
{
var resultVector = Vector.GreaterThanOrEqual(
new Vector <int>(dateValues.Slice(i)),
dateFilterVector
);
resultVector.CopyTo(selectMask, i);
}
for (var i = dateFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = dateValues[i] >= dateFilter;
selectMask[i] = Unsafe.As <bool, int>(ref predicateResult);
//selectMask[i] = dateValues[i] >= dateFilter ? -1 : 0;
}
//selectMask = GetFilterMask<int>(dateValues, Vector.GreaterThanOrEqual, (v, f) => v >= f, dateFilter);
var priceFilterVector = new Vector <float>(5000000);
var priceFilterVectorLimit = recordCount - (recordCount % Vector <float> .Count);
var priceValues = (recordBatch.Column(1) as FloatArray).Values;
for (var i = 0; i < priceFilterVectorLimit; i += Vector <float> .Count)
{
Vector.BitwiseAnd(
new Vector <int>(selectMask, i),
Vector.GreaterThan(
new Vector <float>(priceValues.Slice(i)),
priceFilterVector
)
).CopyTo(selectMask, i);
}
for (var i = priceFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = priceValues[i] > 5000000;
selectMask[i] &= Unsafe.As <bool, int>(ref predicateResult);
//selectMask[i] = selectMask[i] && priceValues[i] > 5000000 ? -1 : 0;
}
byte[] byteSelectMask = VectorizedFiltering.NarrowFilterMask(selectMask);
var stringEncoding = Encoding.ASCII;
var propertyTypeSelectMask = new byte[byteSelectMask.Length];
var propertyTypeFilterList = new string[] { "D", "S", "T" }.Select(x => stringEncoding.GetBytes(x)[0]).ToArray();
var propertyTypeFilterVectorLimit = recordCount - (recordCount % Vector <byte> .Count);
for (var j = 0; j < propertyTypeFilterList.Length; j++)
{
var propertyTypeFilter = propertyTypeFilterList[j];
var propertyTypeFilterVector = new Vector <byte>(propertyTypeFilter);
var propertyTypeValues = (recordBatch.Column(2) as StringArray).Values;
for (var i = 0; i < propertyTypeFilterVectorLimit; i += Vector <byte> .Count)
{
Vector.BitwiseOr(
new Vector <byte>(propertyTypeSelectMask, i),
Vector.GreaterThan(
new Vector <byte>(propertyTypeValues.Slice(i)),
propertyTypeFilterVector
)
).CopyTo(propertyTypeSelectMask, i);
}
for (var i = propertyTypeFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = propertyTypeValues[i] == propertyTypeFilter;
propertyTypeSelectMask[i] |= Unsafe.As <bool, byte>(ref predicateResult);
//byteSelectMask[i] = tenureValues[i] == tenureFilter ? -1 : 0;
}
//for (var i = 0; i < recordCount; i++)
//{
// selectMask[i] = selectMask[i] && propertyTypeFilter.Contains(propertyTypeValues[i]);
//}
}
for (var i = 0; i < propertyTypeFilterVectorLimit; i += Vector <byte> .Count)
{
Vector.BitwiseAnd(
new Vector <byte>(byteSelectMask, i),
new Vector <byte>(propertyTypeSelectMask, i)
).CopyTo(byteSelectMask, i);
}
for (var i = propertyTypeFilterVectorLimit; i < recordCount; i++)
{
byteSelectMask[i] &= propertyTypeSelectMask[i];
}
var tenureFilter = stringEncoding.GetBytes("F")[0];
var tenureFilterVector = new Vector <byte>(tenureFilter);
var tenureFilterVectorLimit = recordCount - (recordCount % Vector <byte> .Count);
var tenureValues = (recordBatch.Column(3) as StringArray).Values;
for (var i = 0; i < tenureFilterVectorLimit; i += Vector <byte> .Count)
{
Vector.BitwiseAnd(
new Vector <byte>(byteSelectMask, i),
Vector.GreaterThan(
new Vector <byte>(tenureValues.Slice(i)),
tenureFilterVector
)
).CopyTo(byteSelectMask, i);
}
for (var i = tenureFilterVectorLimit; i < recordCount; i++)
{
var predicateResult = tenureValues[i] == tenureFilter;
byteSelectMask[i] &= Unsafe.As <bool, byte>(ref predicateResult);
//byteSelectMask[i] = byteSelectMask[i] && tenureValues[i] == tenureFilter ? -1 : 0;
}
var countVector = Vector <byte> .Zero;
for (var i = 0; i < tenureFilterVectorLimit; i += Vector <byte> .Count)
{
countVector += new Vector <byte>(byteSelectMask, i) & Vector <byte> .One;
}
var count = 0;
for (var i = 0; i < Vector <byte> .Count; i++)
{
count += countVector[i];
}
for (var i = tenureFilterVectorLimit; i < recordCount; i++)
{
count += byteSelectMask[i] & 1;
}
#if LOG
Console.WriteLine("Found {0} records", count);
#endif
return(count);
//return selectMask.Count(v => v == -1);
}