/// <exception cref="System.IO.IOException"/>
public virtual bool Next()
{
if (Size() == 0)
{
ResetKeyValue();
return(false);
}
if (minSegment != null)
{
//minSegment is non-null for all invocations of next except the first
//one. For the first invocation, the priority queue is ready for use
//but for the subsequent invocations, first adjust the queue
AdjustPriorityQueue(minSegment);
if (Size() == 0)
{
minSegment = null;
ResetKeyValue();
return(false);
}
}
minSegment = Top();
long startPos = minSegment.GetReader().bytesRead;
key = minSegment.GetKey();
if (!minSegment.InMemory())
{
//When we load the value from an inmemory segment, we reset
//the "value" DIB in this class to the inmem segment's byte[].
//When we load the value bytes from disk, we shouldn't use
//the same byte[] since it would corrupt the data in the inmem
//segment. So we maintain an explicit DIB for value bytes
//obtained from disk, and if the current segment is a disk
//segment, we reset the "value" DIB to the byte[] in that (so
//we reuse the disk segment DIB whenever we consider
//a disk segment).
minSegment.GetValue(diskIFileValue);
value.Reset(diskIFileValue.GetData(), diskIFileValue.GetLength());
}
else
{
minSegment.GetValue(value);
}
long endPos = minSegment.GetReader().bytesRead;
totalBytesProcessed += endPos - startPos;
mergeProgress.Set(totalBytesProcessed * progPerByte);
return(true);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reset()
{
// Create a new segment for the previously written records only if we
// are not already in the reset mode
if (!inReset)
{
if (fileCache.isActive)
{
fileCache.CreateInDiskSegment();
}
else
{
memCache.CreateInMemorySegment();
}
}
inReset = true;
// Reset the segments to the correct position from where the next read
// should begin.
for (int i = 0; i < segmentList.Count; i++)
{
Merger.Segment <K, V> s = segmentList[i];
if (s.InMemory())
{
int offset = (i == 0) ? firstSegmentOffset : 0;
s.GetReader().Reset(offset);
}
else
{
s.CloseReader();
if (i == 0)
{
s.ReinitReader(firstSegmentOffset);
s.GetReader().DisableChecksumValidation();
}
}
}
currentKVOffset = firstSegmentOffset;
nextKVOffset = -1;
readSegmentIndex = 0;
hasMore = false;
lastSegmentEOF = false;
Log.Debug("Reset - First segment offset is " + firstSegmentOffset + " Segment List Size is "
+ segmentList.Count);
}
/// <exception cref="System.IO.IOException"/>
public virtual bool HasNext()
{
if (lastSegmentEOF)
{
return(false);
}
// We read the next KV from the cache to decide if there is any left.
// Since hasNext can be called several times before the actual call to
// next(), we use hasMore to avoid extra reads. hasMore is set to false
// when the user actually consumes this record in next()
if (hasMore)
{
return(true);
}
Merger.Segment <K, V> seg = segmentList[readSegmentIndex];
// Mark the current position. This would be set to currentKVOffset
// when the user consumes this record in next().
nextKVOffset = (int)seg.GetActualPosition();
if (seg.NextRawKey())
{
currentKey = seg.GetKey();
seg.GetValue(currentValue);
hasMore = true;
return(true);
}
else
{
if (!seg.InMemory())
{
seg.CloseReader();
}
}
// If this is the last segment, mark the lastSegmentEOF flag and return
if (readSegmentIndex == segmentList.Count - 1)
{
nextKVOffset = -1;
lastSegmentEOF = true;
return(false);
}
nextKVOffset = 0;
readSegmentIndex++;
Merger.Segment <K, V> nextSegment = segmentList[readSegmentIndex];
// We possibly are moving from a memory segment to a disk segment.
// Reset so that we do not corrupt the in-memory segment buffer.
// See HADOOP-5494
if (!nextSegment.InMemory())
{
currentValue.Reset(currentDiskValue.GetData(), currentDiskValue.GetLength());
nextSegment.Init(null);
}
if (nextSegment.NextRawKey())
{
currentKey = nextSegment.GetKey();
nextSegment.GetValue(currentValue);
hasMore = true;
return(true);
}
else
{
throw new IOException("New segment did not have even one K/V");
}
}