private static void DumpIndex <TKey, TVal>(string label, MemoryIndex <TKey> index, Func <TKey, int, TVal> orderBy, IComparer <TVal> comparer = null, bool heatMaps = false)
{
comparer = comparer ?? Comparer <TVal> .Default;
int total = index.Statistics.Values.Sum();
long totalLegacy = 0;
int[] map = new int[100];
double r = (double)(map.Length - 1) / total;
Log("__{0}__", label);
Log("| Indexed Value | Count | Total % | Words | Lit% | 1-Bits | Word% | Bitmap | ratio % | Legacy | ratio % |" + (heatMaps ? " HeatMap |" : ""));
Log("|:------------------------|-------:|--------:|------:|-------:|-------:|-------:|---------:|--------:|----------:|--------:|" + (heatMaps ? ":-----------------------------------------------------------------------|" : ""));
foreach (var kv in index.Values.OrderBy((kv) => orderBy(kv.Key, index.Count(kv.Key)), comparer))
{
var t = STuple.Create(kv.Key);
var tk = TuPack.Pack(t);
kv.Value.GetStatistics(out int bits, out int words, out int literals, out int _, out double ratio);
long legacyIndexSize = 0; // size estimate of a regular FDB index (..., "Value", GUID) = ""
Array.Clear(map, 0, map.Length);
foreach (var p in kv.Value.GetView())
{
map[(int)(r * p)]++;
legacyIndexSize += 3 + tk.Count + 17;
}
totalLegacy += legacyIndexSize;
int bytes = kv.Value.ToSlice().Count;
Log(string.Format(
CultureInfo.InvariantCulture,
"| {0,-24}| {1,6:N0} | {2,6:N2}% | {3,5:N0} | {4,5:N1}% | {5,6:N0} | {6,6:N2} | {7,8:N0} | {8,6:N2}% | {9,9:N0} | {10,6:N2}% |" + (heatMaps ? " `{11}` |" : ""),
/*0*/ t,
/*1*/ index.Count(kv.Key),
/*2*/ 100.0 * index.Frequency(kv.Key),
/*3*/ words,
/*4*/ (100.0 * literals) / words,
/*5*/ bits,
/*6*/ 1.0 * bits / words,
/*7*/ bytes,
/*8*/ 100.0 * ratio,
/*9*/ legacyIndexSize,
/*A*/ (100.0 * bytes) / legacyIndexSize,
/*B*/ heatMaps ? MakeHeatMap(map) : ""
));
}
Log(string.Format(
CultureInfo.InvariantCulture,
"> {0:N0} distinct value(s), {1:N0} document(s), {2:N0} bitmap bytes, {3:N0} legacy bytes",
index.Values.Count,
total,
index.Values.Values.Sum(x => x.ToSlice().Count),
totalLegacy
));
}
private static Slice MakeKey(IKeySubspace folder, object key)
{
if (key is IVarTuple t)
{
return(folder[TuPack.Pack(t)]);
}
else if (key is string s)
{
return(folder[Slice.FromStringUtf8(s)]);
}
else
{
throw new FormatException("Unsupported key type: " + key);
}
}
public KeyValuePair <IVarTuple, Slice>[] Split(List <KeyValuePair <string, IVarTuple> > document)
{
if (document == null)
{
throw new ArgumentNullException(nameof(document));
}
return(document
// don't include the id
.Where(kvp => !m_keyComparer.Equals(kvp.Key, this.IdName))
// convert into tuples
.Select(kvp => new KeyValuePair <IVarTuple, Slice>(
STuple.Create(kvp.Key),
TuPack.Pack(kvp.Value)
))
.ToArray());
}
public static FdbQueryRangeExpression RangeStartsWith(IVarTuple tuple, FdbRangeOptions options = null)
{
return(RangeStartsWith(TuPack.Pack(tuple), options));
}