/// <exception cref="System.IO.IOException"/>
public virtual void ReportNextRecordRange(TaskAttemptID taskAttemptID, SortedRanges.Range
range)
{
// This is used when the feature of skipping records is enabled.
// This call exists as a hadoop mapreduce legacy wherein all changes in
// counters/progress/phase/output-size are reported through statusUpdate()
// call but not the next record range information.
throw new IOException("Not yet implemented.");
}
/// <summary>Add the range indices.</summary>
/// <remarks>
/// Add the range indices. It is ensured that the added range
/// doesn't overlap the existing ranges. If it overlaps, the
/// existing overlapping ranges are removed and a single range
/// having the superset of all the removed ranges and this range
/// is added.
/// If the range is of 0 length, doesn't do anything.
/// </remarks>
/// <param name="range">Range to be added.</param>
internal virtual void Add(SortedRanges.Range range)
{
lock (this)
{
if (range.IsEmpty())
{
return;
}
long startIndex = range.GetStartIndex();
long endIndex = range.GetEndIndex();
//make sure that there are no overlapping ranges
ICollection <SortedRanges.Range> headSet = ranges.HeadSet(range);
if (headSet.Count > 0)
{
SortedRanges.Range previousRange = headSet.Last();
Log.Debug("previousRange " + previousRange);
if (startIndex < previousRange.GetEndIndex())
{
//previousRange overlaps this range
//remove the previousRange
if (ranges.Remove(previousRange))
{
indicesCount -= previousRange.GetLength();
}
//expand this range
startIndex = previousRange.GetStartIndex();
endIndex = endIndex >= previousRange.GetEndIndex() ? endIndex : previousRange.GetEndIndex
();
}
}
IEnumerator <SortedRanges.Range> tailSetIt = ranges.TailSet(range).GetEnumerator();
while (tailSetIt.HasNext())
{
SortedRanges.Range nextRange = tailSetIt.Next();
Log.Debug("nextRange " + nextRange + " startIndex:" + startIndex + " endIndex:"
+ endIndex);
if (endIndex >= nextRange.GetStartIndex())
{
//nextRange overlaps this range
//remove the nextRange
tailSetIt.Remove();
indicesCount -= nextRange.GetLength();
if (endIndex < nextRange.GetEndIndex())
{
//expand this range
endIndex = nextRange.GetEndIndex();
break;
}
}
else
{
break;
}
}
Add(startIndex, endIndex);
}
}
private void Add(long start, long end)
{
if (end > start)
{
SortedRanges.Range recRange = new SortedRanges.Range(start, end - start);
ranges.AddItem(recRange);
indicesCount += recRange.GetLength();
Log.Debug("added " + recRange);
}
}
private void DoNext()
{
next++;
Log.Debug("currentIndex " + next + " " + range);
SkipIfInRange();
while (next >= range.GetEndIndex() && rangeIterator.HasNext())
{
range = rangeIterator.Next();
SkipIfInRange();
}
}
public override string ToString()
{
StringBuilder sb = new StringBuilder();
IEnumerator <SortedRanges.Range> it = ranges.GetEnumerator();
while (it.HasNext())
{
SortedRanges.Range range = it.Next();
sb.Append(range.ToString() + "\n");
}
return(sb.ToString());
}
/// <exception cref="System.IO.IOException"/>
public virtual void Write(DataOutput @out)
{
lock (this)
{
@out.WriteLong(indicesCount);
@out.WriteInt(ranges.Count);
IEnumerator <SortedRanges.Range> it = ranges.GetEnumerator();
while (it.HasNext())
{
SortedRanges.Range range = it.Next();
range.Write(@out);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
lock (this)
{
indicesCount = @in.ReadLong();
ranges = new TreeSet <SortedRanges.Range>();
int size = @in.ReadInt();
for (int i = 0; i < size; i++)
{
SortedRanges.Range range = new SortedRanges.Range();
range.ReadFields(@in);
ranges.AddItem(range);
}
}
}
/// <summary>Update the status of the task.</summary>
/// <param name="status">updated status</param>
internal virtual void StatusUpdate(Org.Apache.Hadoop.Mapred.TaskStatus status)
{
lock (this)
{
SetProgress(status.GetProgress());
this.runState = status.GetRunState();
SetStateString(status.GetStateString());
this.nextRecordRange = status.GetNextRecordRange();
SetDiagnosticInfo(status.GetDiagnosticInfo());
if (status.GetStartTime() > 0)
{
this.SetStartTime(status.GetStartTime());
}
if (status.GetFinishTime() > 0)
{
this.SetFinishTime(status.GetFinishTime());
}
this.phase = status.GetPhase();
this.counters = status.GetCounters();
this.outputSize = status.outputSize;
}
}
/// <exception cref="System.IO.IOException"/>
public override void ReportNextRecordRange(TaskAttemptID taskid, SortedRanges.Range
range)
{
}
/// <summary>Report the record range which is going to process next by the Task.</summary>
/// <param name="taskid">the id of the task involved</param>
/// <param name="range">the range of record sequence nos</param>
/// <exception cref="System.IO.IOException"/>
public abstract void ReportNextRecordRange(TaskAttemptID taskid, SortedRanges.Range
range);
// Ignore for now
/// <exception cref="System.IO.IOException"/>
public override void ReportNextRecordRange(TaskAttemptID taskid, SortedRanges.Range
range)
{
LocalJobRunner.Log.Info("Task " + taskid + " reportedNextRecordRange " + range);
}
/// <summary>Set the next record range which is going to be processed by Task.</summary>
/// <param name="nextRecordRange"/>
public virtual void SetNextRecordRange(SortedRanges.Range nextRecordRange)
{
this.nextRecordRange = nextRecordRange;
}
