/// <summary>
/// Attempts to create a feature from a ulong array. The intent
/// is that this will handle query ID.
/// </summary>
public static TsvFeature CreateFeatureFromQueryId(Dataset.DatasetSkeleton skel)
{
Dictionary <uint, int> uvalToOrder = new Dictionary <uint, int>();
foreach (uint uintQid in skel.QueryIds.Select(qid => (uint)qid).Distinct().OrderBy(x => x))
{
uvalToOrder[uintQid] = uvalToOrder.Count;
}
IntArray bins = IntArray.New(
skel.NumDocs, IntArrayType.Dense, IntArray.NumBitsNeeded(uvalToOrder.Count),
skel.QueryIds.SelectMany((qid, i) =>
Enumerable.Repeat(uvalToOrder[(uint)qid], skel.Boundaries[i + 1] - skel.Boundaries[i])));
uint[] valueMap = uvalToOrder.Keys.OrderBy(x => x).ToArray(uvalToOrder.Count);
return(new TsvFeature(bins, valueMap, "m:QueryId"));
}
private static IntArray ConcatBins(TsvFeature[] parts, uint[] concatValueMap)
{
using (Timer.Time(TimerEvent.ConcatBins))
{
int length = parts.Sum(x => x.Length);
IntArrayBits bitsPerItem = IntArray.NumBitsNeeded(concatValueMap.Length);
DenseIntArray concatBins = (DenseIntArray)IntArray.New(length, IntArrayType.Dense, bitsPerItem);
int pos = 0;
for (int partIndex = 0; partIndex < parts.Length; ++partIndex)
{
IntArray bins = parts[partIndex].Bins;
if (concatValueMap.Length == parts[partIndex].ValueMap.Length)
{
foreach (int bin in bins)
{
concatBins[pos++] = bin;
}
}
else
{
int[] binMap = MakeBinMap(parts[partIndex]._valueMap, concatValueMap);
foreach (int bin in bins)
{
concatBins[pos++] = binMap[bin];
}
}
}
if (bitsPerItem != IntArrayBits.Bits0 && parts.All(x => x.Bins is DeltaSparseIntArray))
{
return(new DeltaSparseIntArray(length, bitsPerItem, concatBins));
}
else
{
return(concatBins);
}
}
}
/// <summary>
/// Concatenates an array of features into one long feature
/// </summary>
/// <param name="parts">An array of features</param>
/// <returns>A concatenated feature</returns>
public static TsvFeature Concat(TsvFeature[] parts)
{
IntArrayBits bitsPerItem = IntArrayBits.Bits0;
if (parts.Length == 1)
{
bitsPerItem = IntArray.NumBitsNeeded(parts[0].ValueMap.Length);
if (bitsPerItem == parts[0].Bins.BitsPerItem)
{
return(parts[0]);
}
IntArray b = parts[0].Bins;
IntArray newBins = IntArray.New(b.Length, b.Type, bitsPerItem, b);
return(new TsvFeature(newBins, parts[0].ValueMap, parts[0]._name));
}
uint[] concatValueMap = Algorithms.MergeSortedUniqued(parts.Select(x => x.ValueMap).ToArray());
bitsPerItem = IntArray.NumBitsNeeded(concatValueMap.Length);
IntArray concatBins = ConcatBins(parts, concatValueMap);
return(new TsvFeature(concatBins, concatValueMap, parts[0]._name));
}
/// <summary>
/// Finds the most space efficient representation of the feature
/// (with slight slack cut for dense features). The behavior of
/// this method depends upon the static value <see cref="CompatibilityLevel"/>.
/// </summary>
/// <param name="workarray">Should be non-null if you want it to
/// consider segment arrays.</param>
/// <returns>Returns a more space efficient version of the array,
/// or the item itself if that is impossible, somehow.</returns>
public IntArray Compress(uint[] workarray = null)
{
int maxval = 0;
int zerocount = 0;
int runs = 0;
int last = -1;
int overflows = 0;
int zoverflows = 0;
int runnow = 0; // The longest run of having the same value.
int len = Length;
IIntArrayForwardIndexer ind = GetIndexer();
for (int i = 0; i < len; ++i)
{
int val = ind[i];
if (workarray != null)
{
workarray[i] = (uint)val;
}
if (val == 0)
{
zerocount++;
}
else if (val > maxval)
{
maxval = val;
}
if (last == val)
{
runs++;
if (++runnow > byte.MaxValue)
{
// We have 256 items in a row the same.
overflows++;
if (val == 0)
{
zoverflows++;
}
runnow = 0;
}
}
last = val;
}
// Estimate the costs of the available options.
IntArrayBits classicBits = IntArray.NumBitsNeeded(maxval + 1);
long denseBits = (long)classicBits * (long)Length;
long sparseBits = (long)(Math.Max((int)classicBits, 8) + 8) * (long)(Length - zerocount + zoverflows);
long rleBits = (long)(classicBits + 8) * (long)(Length - runs + overflows);
long segBits = long.MaxValue;
int segTransitions = 0;
if (workarray != null)
{
int bits = SegmentIntArray.BitsForValue((uint)maxval);
if (bits <= 21)
{
SegmentIntArray.SegmentFindOptimalPath(workarray, Length,
bits, out segBits, out segTransitions);
}
}
if ((IntArray.CompatibilityLevel & 0x4) == 0)
{
rleBits = long.MaxValue;
}
long bestCost = Math.Min(Math.Min(Math.Min(denseBits, sparseBits), rleBits), segBits);
IntArrayType bestType = IntArrayType.Dense;
if (bestCost >= denseBits * 98 / 100)
{
// Cut the dense bits a wee bit of slack.
}
else if (bestCost == sparseBits)
{
bestType = IntArrayType.Sparse;
}
else if (bestCost == rleBits)
{
bestType = IntArrayType.Repeat;
}
else
{
bestType = IntArrayType.Segmented;
}
if (bestType == Type && classicBits == BitsPerItem)
{
return(this);
}
IntArray bins = null;
if (bestType != IntArrayType.Segmented)
{
bins = IntArray.New(Length, bestType, classicBits, this);
}
else
{
bins = SegmentIntArray.FromWorkArray(workarray, Length, segBits, segTransitions);
}
return(bins);
}