public override MergeSpecification FindMerges(MergeTrigger?mergeTrigger, SegmentInfos infos)
{
if (Verbose())
{
Message("findMerges: " + infos.Size() + " segments");
}
if (infos.Size() == 0)
{
return(null);
}
ICollection <SegmentCommitInfo> merging = Writer.Get().MergingSegments;
ICollection <SegmentCommitInfo> toBeMerged = new HashSet <SegmentCommitInfo>();
List <SegmentCommitInfo> infosSorted = new List <SegmentCommitInfo>(infos.AsList());
infosSorted.Sort(new SegmentByteSizeDescending(this));
// Compute total index bytes & print details about the index
long totIndexBytes = 0;
long minSegmentBytes = long.MaxValue;
foreach (SegmentCommitInfo info in infosSorted)
{
long segBytes = Size(info);
if (Verbose())
{
string extra = merging.Contains(info) ? " [merging]" : "";
if (segBytes >= MaxMergedSegmentBytes / 2.0)
{
extra += " [skip: too large]";
}
else if (segBytes < FloorSegmentBytes)
{
extra += " [floored]";
}
Message(" seg=" + Writer.Get().SegString(info) + " size=" + String.Format(CultureInfo.InvariantCulture, "{0:0.00}", segBytes / 1024 / 1024.0) + " MB" + extra);
}
minSegmentBytes = Math.Min(segBytes, minSegmentBytes);
// Accum total byte size
totIndexBytes += segBytes;
}
// If we have too-large segments, grace them out
// of the maxSegmentCount:
int tooBigCount = 0;
while (tooBigCount < infosSorted.Count && Size(infosSorted[tooBigCount]) >= MaxMergedSegmentBytes / 2.0)
{
totIndexBytes -= Size(infosSorted[tooBigCount]);
tooBigCount++;
}
minSegmentBytes = FloorSize(minSegmentBytes);
// Compute max allowed segs in the index
long levelSize = minSegmentBytes;
long bytesLeft = totIndexBytes;
double allowedSegCount = 0;
while (true)
{
double segCountLevel = bytesLeft / (double)levelSize;
if (segCountLevel < SegsPerTier)
{
allowedSegCount += Math.Ceiling(segCountLevel);
break;
}
allowedSegCount += SegsPerTier;
bytesLeft -= (long)(SegsPerTier * levelSize);
levelSize *= MaxMergeAtOnce_Renamed;
}
int allowedSegCountInt = (int)allowedSegCount;
MergeSpecification spec = null;
// Cycle to possibly select more than one merge:
while (true)
{
long mergingBytes = 0;
// Gather eligible segments for merging, ie segments
// not already being merged and not already picked (by
// prior iteration of this loop) for merging:
IList <SegmentCommitInfo> eligible = new List <SegmentCommitInfo>();
for (int idx = tooBigCount; idx < infosSorted.Count; idx++)
{
SegmentCommitInfo info = infosSorted[idx];
if (merging.Contains(info))
{
mergingBytes += info.SizeInBytes();
}
else if (!toBeMerged.Contains(info))
{
eligible.Add(info);
}
}
bool maxMergeIsRunning = mergingBytes >= MaxMergedSegmentBytes;
if (Verbose())
{
Message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.Count + " (eligible count=" + eligible.Count + ") tooBigCount=" + tooBigCount);
}
if (eligible.Count == 0)
{
return(spec);
}
if (eligible.Count >= allowedSegCountInt)
{
// OK we are over budget -- find best merge!
MergeScore bestScore = null;
IList <SegmentCommitInfo> best = null;
bool bestTooLarge = false;
long bestMergeBytes = 0;
// Consider all merge starts:
for (int startIdx = 0; startIdx <= eligible.Count - MaxMergeAtOnce_Renamed; startIdx++)
{
long totAfterMergeBytes = 0;
IList <SegmentCommitInfo> candidate = new List <SegmentCommitInfo>();
bool hitTooLarge = false;
for (int idx = startIdx; idx < eligible.Count && candidate.Count < MaxMergeAtOnce_Renamed; idx++)
{
SegmentCommitInfo info = eligible[idx];
long segBytes = Size(info);
if (totAfterMergeBytes + segBytes > MaxMergedSegmentBytes)
{
hitTooLarge = true;
// NOTE: we continue, so that we can try
// "packing" smaller segments into this merge
// to see if we can get closer to the max
// size; this in general is not perfect since
// this is really "bin packing" and we'd have
// to try different permutations.
continue;
}
candidate.Add(info);
totAfterMergeBytes += segBytes;
}
MergeScore score = Score(candidate, hitTooLarge, mergingBytes);
if (Verbose())
{
Message(" maybe=" + Writer.Get().SegString(candidate) + " score=" + score.Score + " " + score.Explanation + " tooLarge=" + hitTooLarge + " size=" + string.Format(CultureInfo.InvariantCulture, "%.3f MB", totAfterMergeBytes / 1024.0 / 1024.0));
}
// If we are already running a max sized merge
// (maxMergeIsRunning), don't allow another max
// sized merge to kick off:
if ((bestScore == null || score.Score < bestScore.Score) && (!hitTooLarge || !maxMergeIsRunning))
{
best = candidate;
bestScore = score;
bestTooLarge = hitTooLarge;
bestMergeBytes = totAfterMergeBytes;
}
}
if (best != null)
{
if (spec == null)
{
spec = new MergeSpecification();
}
OneMerge merge = new OneMerge(best);
spec.Add(merge);
foreach (SegmentCommitInfo info in merge.Segments)
{
toBeMerged.Add(info);
}
if (Verbose())
{
Message(" add merge=" + Writer.Get().SegString(merge.Segments) + " size=" + string.Format(CultureInfo.InvariantCulture, "%.3f MB", bestMergeBytes / 1024.0 / 1024.0) + " score=" + string.Format(CultureInfo.InvariantCulture, "%.3f", bestScore.Score) + " " + bestScore.Explanation + (bestTooLarge ? " [max merge]" : ""));
}
}
else
{
return(spec);
}
}
else
{
return(spec);
}
}
}
public MergeMatch(MergeScore score)
{
Source = score.Source.Instance;
Destination = score.Destination.Instance;
}