/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal CombineFileRecordReaderWrapper(FileInputFormat <K, V> inputFormat
, CombineFileSplit split, TaskAttemptContext context, int idx)
{
fileSplit = new FileSplit(split.GetPath(idx), split.GetOffset(idx), split.GetLength
(idx), split.GetLocations());
delegate_ = inputFormat.CreateRecordReader(fileSplit, context);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
this.split = (CombineFileSplit)split;
this.context = context;
if (null != this.curReader)
{
this.curReader.Initialize(split, context);
}
}
/// <summary>
/// A generic RecordReader that can hand out different recordReaders
/// for each chunk in the CombineFileSplit.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public CombineFileRecordReader(CombineFileSplit split, TaskAttemptContext context
, Type rrClass)
{
this.split = split;
this.context = context;
this.idx = 0;
this.curReader = null;
this.progress = 0;
try
{
rrConstructor = rrClass.GetDeclaredConstructor(constructorSignature);
}
catch (Exception e)
{
throw new RuntimeException(rrClass.FullName + " does not have valid constructor",
e);
}
InitNextRecordReader();
}
/// <summary>
/// Create a single split from the list of blocks specified in validBlocks
/// Add this new split into splitList.
/// </summary>
private void AddCreatedSplit(IList <InputSplit> splitList, ICollection <string> locations
, AList <CombineFileInputFormat.OneBlockInfo> validBlocks)
{
// create an input split
Path[] fl = new Path[validBlocks.Count];
long[] offset = new long[validBlocks.Count];
long[] length = new long[validBlocks.Count];
for (int i = 0; i < validBlocks.Count; i++)
{
fl[i] = validBlocks[i].onepath;
offset[i] = validBlocks[i].offset;
length[i] = validBlocks[i].length;
}
// add this split to the list that is returned
CombineFileSplit thissplit = new CombineFileSplit(fl, offset, length, Sharpen.Collections.ToArray
(locations, new string[0]));
splitList.AddItem(thissplit);
}
public virtual void TestProgressIsReportedIfInputASeriesOfEmptyFiles()
{
JobConf conf = new JobConf();
Path[] paths = new Path[3];
FilePath[] files = new FilePath[3];
long[] fileLength = new long[3];
try
{
for (int i = 0; i < 3; i++)
{
FilePath dir = new FilePath(outDir.ToString());
dir.Mkdir();
files[i] = new FilePath(dir, "testfile" + i);
FileWriter fileWriter = new FileWriter(files[i]);
fileWriter.Flush();
fileWriter.Close();
fileLength[i] = i;
paths[i] = new Path(outDir + "/testfile" + i);
}
CombineFileSplit combineFileSplit = new CombineFileSplit(paths, fileLength);
TaskAttemptID taskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
Task.TaskReporter reporter = Org.Mockito.Mockito.Mock <Task.TaskReporter>();
TaskAttemptContextImpl taskAttemptContext = new TaskAttemptContextImpl(conf, taskAttemptID
, reporter);
CombineFileRecordReader cfrr = new CombineFileRecordReader(combineFileSplit, taskAttemptContext
, typeof(TestCombineFileRecordReader.TextRecordReaderWrapper));
cfrr.Initialize(combineFileSplit, taskAttemptContext);
Org.Mockito.Mockito.Verify(reporter).Progress();
NUnit.Framework.Assert.IsFalse(cfrr.NextKeyValue());
Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Times(3)).Progress();
}
finally
{
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public SequenceFileRecordReaderWrapper(CombineFileSplit split, TaskAttemptContext
context, int idx)
: base(new SequenceFileInputFormat <K, V>(), split, context, idx)
{
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public TextRecordReaderWrapper(CombineFileSplit split, TaskAttemptContext context
, int idx)
: base(new TextInputFormat(), split, context, idx)
{
}