public NHotFeatureFlock(DenseIntArray values, byte[] deltas, int len,
int[] hotFeatureStarts, double[][] binUpperBounds)
: base(new DeltaSparseIntArray(values, deltas, len), hotFeatureStarts, binUpperBounds)
{
_values = values;
_deltas = deltas;
}
public DeltaRepeatIntArray(byte[] buffer, ref int position)
{
_length = buffer.ToInt(ref position);
_deltasActualLength = position;
_deltasActualLength = buffer.ToInt(ref _deltasActualLength);
_deltas = buffer.ToByteArray(ref position);
_values = IntArray.New(buffer, ref position) as DenseIntArray;
}
/// <summary>
/// Construct a sparse int array from index, value pairs.
/// </summary>
/// <param name="length">The total length of the constructed array.</param>
/// <param name="bitsPerItem">The number of bits required to store the values.</param>
/// <param name="nonZeroValues">An ordered enumerable of (index,value) pairs.
/// Each index should be strictly increasing as the iterable proceeds.</param>
public DeltaSparseIntArray(int length, IntArrayBits bitsPerItem, IEnumerable <KeyValuePair <int, int> > nonZeroValues)
{
using (Timer.Time(TimerEvent.SparseConstruction))
{
List <int> tempValueList = new List <int>();
List <byte> tempDeltaList = new List <byte>();
int currentIndex = 0;
foreach (KeyValuePair <int, int> pair in nonZeroValues)
{
int index = pair.Key;
int value = pair.Value;
if (index <= currentIndex && (index < 0 || tempValueList.Count > 0))
{
throw Contracts.Except("index {0} occurred after {1}", index, currentIndex);
}
while (index - currentIndex > byte.MaxValue)
{
tempDeltaList.Add(byte.MaxValue);
tempValueList.Add(0);
currentIndex += byte.MaxValue;
}
tempDeltaList.Add((byte)(index - currentIndex));
tempValueList.Add(value);
currentIndex = index;
}
// Add the final chunks of 0's if it ended early
while (length - currentIndex > byte.MaxValue)
{
tempDeltaList.Add(byte.MaxValue);
tempValueList.Add(0);
currentIndex += byte.MaxValue;
}
if (currentIndex >= length && currentIndex > 0)
{
throw Contracts.Except("Index {0} inconsistent with length {1}", currentIndex, length);
}
_length = length;
// It is faster not to use a 4-bit dense array here. The memory difference is minor, since it's just
// the sparse values that are saved on.
// TODO: Implement a special iterator for 4-bit array, and change this code to use the iterator, which
// may be faster
if (bitsPerItem == IntArrayBits.Bits0)
{
throw Contracts.Except("Use dense arrays for 0 bits");
}
if (bitsPerItem <= IntArrayBits.Bits8)
{
bitsPerItem = IntArrayBits.Bits8;
}
_values = IntArray.New(tempValueList.Count, IntArrayType.Dense, bitsPerItem, tempValueList) as DenseIntArray;
_deltas = tempDeltaList.ToArray();
}
}
public DeltaRepeatIntArray(int length, IntArrayBits bitsPerItem, IEnumerable <int> values)
{
using (Timer.Time(TimerEvent.SparseConstruction))
{
List <int> tempValueList = new List <int>();
List <byte> tempDeltaList = new List <byte>();
_length = 0;
byte delta = 0;
int lastVal = -1;
foreach (int val in values)
{
if (val != lastVal || delta == byte.MaxValue)
{
tempValueList.Add(val);
lastVal = val;
if (_length != 0)
{
tempDeltaList.Add(delta);
}
delta = 0;
}
++delta;
++_length;
}
if (delta > 0)
{
tempDeltaList.Add(delta);
}
if (_length != length)
{
throw Contracts.Except("Length provided to repeat vector is inconsistent with value enumeration");
}
// It is faster not to use a 4-bit dense array here. The memory difference is minor, since it's just
// the sparse values that are saved on.
// TODO: Implement a special iterator for 4-bit array, and change this code to use the iterator, which
// may be faster
if (bitsPerItem == IntArrayBits.Bits0)
{
throw Contracts.Except("Use dense arrays for 0 bits");
}
if (bitsPerItem <= IntArrayBits.Bits8)
{
bitsPerItem = IntArrayBits.Bits8;
}
_values = IntArray.New(tempValueList.Count, IntArrayType.Dense, bitsPerItem, tempValueList) as DenseIntArray;
_deltas = tempDeltaList.ToArray();
}
}
public DeltaSparseIntArray(byte[] buffer, ref int position)
{
_length = buffer.ToInt(ref position);
// REVIEW: The two lines below is as it actually appeared. I have no earthly idea of what
// it was trying to accomplish. It appears to function as a no-op, resulting in no valid results
// inside _deltasActualLength.
//_deltasActualLength = position;
//_deltasActualLength = buffer.ToInt(ref _deltasActualLength);
_deltas = buffer.ToByteArray(ref position);
_values = IntArray.New(buffer, ref position) as DenseIntArray;
}
private DeltaRepeatIntArray(DenseIntArray values, byte[] deltas, int length)
{
_values = values;
_deltas = deltas;
_length = length;
_deltasActualLength = length;
if (BitsPerItem == IntArrayBits.Bits0)
{
throw Contracts.Except("Use dense arrays for 0 bits");
}
}
public DeltaSparseIntArray(DenseIntArray values, byte[] deltas, int length)
{
Contracts.AssertValue(values);
Contracts.AssertValue(deltas);
Contracts.Assert(values.Length == deltas.Length);
Contracts.Assert(deltas.Sum(d => (long)d) < length);
_values = values;
_deltas = deltas;
_length = length;
if (BitsPerItem == IntArrayBits.Bits0)
{
throw Contracts.Except("Use dense arrays for 0 bits");
}
}
private static IntArray ConcatBins(TsvFeature[] parts, uint[] concatValueMap)
{
using (Timer.Time(TimerEvent.ConcatBins))
{
int length = parts.Sum(x => x.Length);
IntArrayBits bitsPerItem = IntArray.NumBitsNeeded(concatValueMap.Length);
DenseIntArray concatBins = (DenseIntArray)IntArray.New(length, IntArrayType.Dense, bitsPerItem);
int pos = 0;
for (int partIndex = 0; partIndex < parts.Length; ++partIndex)
{
IntArray bins = parts[partIndex].Bins;
if (concatValueMap.Length == parts[partIndex].ValueMap.Length)
{
foreach (int bin in bins)
{
concatBins[pos++] = bin;
}
}
else
{
int[] binMap = MakeBinMap(parts[partIndex]._valueMap, concatValueMap);
foreach (int bin in bins)
{
concatBins[pos++] = binMap[bin];
}
}
}
if (bitsPerItem != IntArrayBits.Bits0 && parts.All(x => x.Bins is DeltaSparseIntArray))
{
return(new DeltaSparseIntArray(length, bitsPerItem, concatBins));
}
else
{
return(concatBins);
}
}
}
public override IntArray Clone(IntArrayBits bitsPerItem, IntArrayType type)
{
if (type == IntArrayType.Sparse || type == IntArrayType.Current)
{
if (bitsPerItem <= IntArrayBits.Bits8)
{
bitsPerItem = IntArrayBits.Bits8;
}
DenseIntArray newValues = _values.Clone(bitsPerItem, IntArrayType.Dense) as DenseIntArray;
return(new DeltaSparseIntArray(newValues, _deltas, _length));
}
else
{
DenseIntArray dense = IntArray.New(Length, IntArrayType.Dense, BitsPerItem) as DenseIntArray;
int index = 0;
for (int i = 0; i < _values.Length; ++i)
{
index += _deltas[i];
dense[index] = _values[i];
}
return(dense);
}
}
/// <summary>
/// Constructs an empty (all zero) feature
/// </summary>
/// <param name="name"></param>
/// <param name="length"></param>
public TsvFeature(string name, int length)
: base(DenseIntArray.New(length, IntArrayType.Dense, 0, Enumerable.Repeat(0, length)))
{
_valueMap = new uint[1];
_name = name;
}