public virtual void TestInitNextRecordReader() { JobConf conf = new JobConf(); Path[] paths = new Path[3]; long[] fileLength = new long[3]; FilePath[] files = new FilePath[3]; LongWritable key = new LongWritable(1); Text value = new Text(); try { for (int i = 0; i < 3; i++) { fileLength[i] = i; FilePath dir = new FilePath(outDir.ToString()); dir.Mkdir(); files[i] = new FilePath(dir, "testfile" + i); FileWriter fileWriter = new FileWriter(files[i]); fileWriter.Close(); paths[i] = new Path(outDir + "/testfile" + i); } CombineFileSplit combineFileSplit = new CombineFileSplit(conf, paths, fileLength); Reporter reporter = Org.Mockito.Mockito.Mock <Reporter>(); CombineFileRecordReader cfrr = new CombineFileRecordReader(conf, combineFileSplit , reporter, typeof(TestCombineFileRecordReader.TextRecordReaderWrapper)); Org.Mockito.Mockito.Verify(reporter).Progress(); NUnit.Framework.Assert.IsFalse(cfrr.Next(key, value)); Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Times(3)).Progress(); } finally { FileUtil.FullyDelete(new FilePath(outDir.ToString())); } }
/// <exception cref="System.IO.IOException"/> protected internal CombineFileRecordReaderWrapper(FileInputFormat <K, V> inputFormat , CombineFileSplit split, Configuration conf, Reporter reporter, int idx) { FileSplit fileSplit = new FileSplit(split.GetPath(idx), split.GetOffset(idx), split .GetLength(idx), split.GetLocations()); delegate_ = inputFormat.GetRecordReader(fileSplit, (JobConf)conf, reporter); }
/// <exception cref="System.IO.IOException"/> public virtual InputSplit[] GetSplits(JobConf job, int numSplits) { IList <InputSplit> newStyleSplits = base.GetSplits(Job.GetInstance(job)); InputSplit[] ret = new InputSplit[newStyleSplits.Count]; for (int pos = 0; pos < newStyleSplits.Count; ++pos) { CombineFileSplit newStyleSplit = (CombineFileSplit)newStyleSplits[pos]; ret[pos] = new CombineFileSplit(job, newStyleSplit.GetPaths(), newStyleSplit.GetStartOffsets (), newStyleSplit.GetLengths(), newStyleSplit.GetLocations()); } return(ret); }
/// <summary> /// A generic RecordReader that can hand out different recordReaders /// for each chunk in the CombineFileSplit. /// </summary> /// <exception cref="System.IO.IOException"/> public CombineFileRecordReader(JobConf job, CombineFileSplit split, Reporter reporter , Type rrClass) { this.split = split; this.jc = job; this.reporter = reporter; this.idx = 0; this.curReader = null; this.progress = 0; try { rrConstructor = rrClass.GetDeclaredConstructor(constructorSignature); } catch (Exception e) { throw new RuntimeException(rrClass.FullName + " does not have valid constructor", e); } InitNextRecordReader(); }
/// <exception cref="System.IO.IOException"/> public TextRecordReaderWrapper(CombineFileSplit split, Configuration conf, Reporter reporter, int idx) : base(new TextInputFormat(), split, conf, reporter, idx) { }
/// <exception cref="System.IO.IOException"/> public SequenceFileRecordReaderWrapper(CombineFileSplit split, Configuration conf , Reporter reporter, int idx) : base(new SequenceFileInputFormat <K, V>(), split, conf, reporter, idx) { }