public virtual void TestInitNextRecordReader()
{
JobConf conf = new JobConf();
Path[] paths = new Path[3];
long[] fileLength = new long[3];
FilePath[] files = new FilePath[3];
LongWritable key = new LongWritable(1);
Text value = new Text();
try
{
for (int i = 0; i < 3; i++)
{
fileLength[i] = i;
FilePath dir = new FilePath(outDir.ToString());
dir.Mkdir();
files[i] = new FilePath(dir, "testfile" + i);
FileWriter fileWriter = new FileWriter(files[i]);
fileWriter.Close();
paths[i] = new Path(outDir + "/testfile" + i);
}
CombineFileSplit combineFileSplit = new CombineFileSplit(conf, paths, fileLength);
Reporter reporter = Org.Mockito.Mockito.Mock <Reporter>();
CombineFileRecordReader cfrr = new CombineFileRecordReader(conf, combineFileSplit
, reporter, typeof(TestCombineFileRecordReader.TextRecordReaderWrapper));
Org.Mockito.Mockito.Verify(reporter).Progress();
NUnit.Framework.Assert.IsFalse(cfrr.Next(key, value));
Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Times(3)).Progress();
}
finally
{
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
}
/// <exception cref="System.IO.IOException"/>
protected internal CombineFileRecordReaderWrapper(FileInputFormat <K, V> inputFormat
, CombineFileSplit split, Configuration conf, Reporter reporter, int idx)
{
FileSplit fileSplit = new FileSplit(split.GetPath(idx), split.GetOffset(idx), split
.GetLength(idx), split.GetLocations());
delegate_ = inputFormat.GetRecordReader(fileSplit, (JobConf)conf, reporter);
}
/// <exception cref="System.IO.IOException"/>
public virtual InputSplit[] GetSplits(JobConf job, int numSplits)
{
IList <InputSplit> newStyleSplits = base.GetSplits(Job.GetInstance(job));
InputSplit[] ret = new InputSplit[newStyleSplits.Count];
for (int pos = 0; pos < newStyleSplits.Count; ++pos)
{
CombineFileSplit newStyleSplit = (CombineFileSplit)newStyleSplits[pos];
ret[pos] = new CombineFileSplit(job, newStyleSplit.GetPaths(), newStyleSplit.GetStartOffsets
(), newStyleSplit.GetLengths(), newStyleSplit.GetLocations());
}
return(ret);
}
/// <summary>
/// A generic RecordReader that can hand out different recordReaders
/// for each chunk in the CombineFileSplit.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public CombineFileRecordReader(JobConf job, CombineFileSplit split, Reporter reporter
, Type rrClass)
{
this.split = split;
this.jc = job;
this.reporter = reporter;
this.idx = 0;
this.curReader = null;
this.progress = 0;
try
{
rrConstructor = rrClass.GetDeclaredConstructor(constructorSignature);
}
catch (Exception e)
{
throw new RuntimeException(rrClass.FullName + " does not have valid constructor",
e);
}
InitNextRecordReader();
}
/// <exception cref="System.IO.IOException"/>
public TextRecordReaderWrapper(CombineFileSplit split, Configuration conf, Reporter
reporter, int idx)
: base(new TextInputFormat(), split, conf, reporter, idx)
{
}
/// <exception cref="System.IO.IOException"/>
public SequenceFileRecordReaderWrapper(CombineFileSplit split, Configuration conf
, Reporter reporter, int idx)
: base(new SequenceFileInputFormat <K, V>(), split, conf, reporter, idx)
{
}