private void BuildSortedSet()
{
var comparer = new BuildSortedSetComparer(this);
m_orderedGroups = new JCG.SortedSet <CollectedSearchGroup <TGroupValue> >(comparer);
m_orderedGroups.UnionWith(groupMap.Values);
Debug.Assert(m_orderedGroups.Count > 0);
foreach (FieldComparer fc in comparers)
{
fc.SetBottom(m_orderedGroups.Last().ComparerSlot);
}
}
public virtual void Collect(int doc)
{
//System.out.println("FP.collect doc=" + doc);
// If orderedGroups != null we already have collected N groups and
// can short circuit by comparing this document to the bottom group,
// without having to find what group this document belongs to.
// Even if this document belongs to a group in the top N, we'll know that
// we don't have to update that group.
// Downside: if the number of unique groups is very low, this is
// wasted effort as we will most likely be updating an existing group.
if (m_orderedGroups != null)
{
for (int compIDX = 0; ; compIDX++)
{
int c = reversed[compIDX] * comparers[compIDX].CompareBottom(doc);
if (c < 0)
{
// Definitely not competitive. So don't even bother to continue
return;
}
else if (c > 0)
{
// Definitely competitive.
break;
}
else if (compIDX == compIDXEnd)
{
// Here c=0. If we're at the last comparer, this doc is not
// competitive, since docs are visited in doc Id order, which means
// this doc cannot compete with any other document in the queue.
return;
}
}
}
// TODO: should we add option to mean "ignore docs that
// don't have the group field" (instead of stuffing them
// under null group)?
TGroupValue groupValue = GetDocGroupValue(doc);
if (!groupMap.TryGetValue(groupValue, out CollectedSearchGroup <TGroupValue> group))
{
// First time we are seeing this group, or, we've seen
// it before but it fell out of the top N and is now
// coming back
if (groupMap.Count < topNGroups)
{
// Still in startup transient: we have not
// seen enough unique groups to start pruning them;
// just keep collecting them
// Add a new CollectedSearchGroup:
CollectedSearchGroup <TGroupValue> sg = new CollectedSearchGroup <TGroupValue>();
sg.GroupValue = CopyDocGroupValue(groupValue, default);
sg.ComparerSlot = groupMap.Count;
sg.TopDoc = docBase + doc;
foreach (FieldComparer fc in comparers)
{
fc.Copy(sg.ComparerSlot, doc);
}
groupMap[sg.GroupValue] = sg;
if (groupMap.Count == topNGroups)
{
// End of startup transient: we now have max
// number of groups; from here on we will drop
// bottom group when we insert new one:
BuildSortedSet();
}
return;
}
// We already tested that the document is competitive, so replace
// the bottom group with this new group.
//CollectedSearchGroup<TGroupValue> bottomGroup = orderedGroups.PollLast();
CollectedSearchGroup <TGroupValue> bottomGroup;
UninterruptableMonitor.Enter(m_orderedGroups);
try
{
bottomGroup = m_orderedGroups.Last();
m_orderedGroups.Remove(bottomGroup);
}
finally
{
UninterruptableMonitor.Exit(m_orderedGroups);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(m_orderedGroups.Count == topNGroups - 1);
}
groupMap.Remove(bottomGroup.GroupValue);
// reuse the removed CollectedSearchGroup
bottomGroup.GroupValue = CopyDocGroupValue(groupValue, bottomGroup.GroupValue);
bottomGroup.TopDoc = docBase + doc;
foreach (FieldComparer fc in comparers)
{
fc.Copy(bottomGroup.ComparerSlot, doc);
}
groupMap[bottomGroup.GroupValue] = bottomGroup;
m_orderedGroups.Add(bottomGroup);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(m_orderedGroups.Count == topNGroups);
}
int lastComparerSlot = m_orderedGroups.Last().ComparerSlot;
foreach (FieldComparer fc in comparers)
{
fc.SetBottom(lastComparerSlot);
}
return;
}
// Update existing group:
for (int compIDX = 0; ; compIDX++)
{
FieldComparer fc = comparers[compIDX];
fc.Copy(spareSlot, doc);
int c = reversed[compIDX] * fc.Compare(group.ComparerSlot, spareSlot);
if (c < 0)
{
// Definitely not competitive.
return;
}
else if (c > 0)
{
// Definitely competitive; set remaining comparers:
for (int compIDX2 = compIDX + 1; compIDX2 < comparers.Length; compIDX2++)
{
comparers[compIDX2].Copy(spareSlot, doc);
}
break;
}
else if (compIDX == compIDXEnd)
{
// Here c=0. If we're at the last comparer, this doc is not
// competitive, since docs are visited in doc Id order, which means
// this doc cannot compete with any other document in the queue.
return;
}
}
// Remove before updating the group since lookup is done via comparers
// TODO: optimize this
CollectedSearchGroup <TGroupValue> prevLast;
if (m_orderedGroups != null)
{
UninterruptableMonitor.Enter(m_orderedGroups);
try
{
prevLast = m_orderedGroups.Last();
m_orderedGroups.Remove(group);
}
finally
{
UninterruptableMonitor.Exit(m_orderedGroups);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(m_orderedGroups.Count == topNGroups - 1);
}
}
else
{
prevLast = null;
}
group.TopDoc = docBase + doc;
// Swap slots
int tmp = spareSlot;
spareSlot = group.ComparerSlot;
group.ComparerSlot = tmp;
// Re-add the changed group
if (m_orderedGroups != null)
{
m_orderedGroups.Add(group);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(m_orderedGroups.Count == topNGroups);
}
var newLast = m_orderedGroups.Last();
// If we changed the value of the last group, or changed which group was last, then update bottom:
if (group == newLast || prevLast != newLast)
{
foreach (FieldComparer fc in comparers)
{
fc.SetBottom(newLast.ComparerSlot);
}
}
}
}
private void UpdateNextGroup(int topN, ShardIter <T> shard)
{
while (shard.Iter.MoveNext())
{
ISearchGroup <T> group = shard.Next();
bool isNew = !groupsSeen.TryGetValue(group.GroupValue, out MergedGroup <T> mergedGroup) || mergedGroup == null;
//System.out.println(" next group=" + (group.groupValue == null ? "null" : ((BytesRef) group.groupValue).utf8ToString()) + " sort=" + Arrays.toString(group.sortValues));
if (isNew)
{
// Start a new group:
//System.out.println(" new");
mergedGroup = new MergedGroup <T>(group.GroupValue);
mergedGroup.MinShardIndex = shard.ShardIndex;
Debug.Assert(group.SortValues != null);
mergedGroup.TopValues = group.SortValues;
groupsSeen[group.GroupValue] = mergedGroup;
mergedGroup.IsInQueue = true;
queue.Add(mergedGroup);
}
else if (mergedGroup.IsProcessed)
{
// This shard produced a group that we already
// processed; move on to next group...
continue;
}
else
{
//System.out.println(" old");
bool competes = false;
for (int compIDX = 0; compIDX < groupComp.Comparers.Length; compIDX++)
{
int cmp = groupComp.Reversed[compIDX] * groupComp.Comparers[compIDX].CompareValues(group.SortValues[compIDX],
mergedGroup.TopValues[compIDX]);
if (cmp < 0)
{
// Definitely competes
competes = true;
break;
}
else if (cmp > 0)
{
// Definitely does not compete
break;
}
else if (compIDX == groupComp.Comparers.Length - 1)
{
if (shard.ShardIndex < mergedGroup.MinShardIndex)
{
competes = true;
}
}
}
//System.out.println(" competes=" + competes);
if (competes)
{
// Group's sort changed -- remove & re-insert
if (mergedGroup.IsInQueue)
{
queue.Remove(mergedGroup);
}
mergedGroup.TopValues = group.SortValues;
mergedGroup.MinShardIndex = shard.ShardIndex;
queue.Add(mergedGroup);
mergedGroup.IsInQueue = true;
}
}
mergedGroup.Shards.Add(shard);
break;
}
// Prune un-competitive groups:
while (queue.Count > topN)
{
MergedGroup <T> group = queue.Last();
queue.Remove(group);
//System.out.println("PRUNE: " + group);
group.IsInQueue = false;
}
}