/// <exception cref="System.IO.IOException"/> public virtual RecordReader <object, object> GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { return(new _RecordReader_62()); }
public override RecordReader <K, V> CreateRecordReader(InputSplit ignored, TaskAttemptContext context) { return(new TestWrappedRRClassloader.RR_ClassLoaderChecker <K, V>(context.GetConfiguration ())); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override RecordReader <Text, Text> CreateRecordReader(InputSplit split, TaskAttemptContext context) { return(new RandomTextWriterJob.RandomInputFormat.RandomRecordReader(((FileSplit)split ).GetPath())); }
// abstract method from super class implemented to return null /// <exception cref="System.IO.IOException"/> public override RecordReader <K, V> CreateRecordReader(InputSplit split, TaskAttemptContext context) { return(null); }
/// <exception cref="System.IO.IOException"/> public override RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { return(null); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Initialize(InputSplit split, TaskAttemptContext context) { this.i.Set(((TestInputSampler.SequentialSplit)split).GetInit() - 1); this.maxVal = this.i.Get() + this._enclosing.maxDepth + 1; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override RecordReader <IntWritable, NullWritable> CreateRecordReader(InputSplit split, TaskAttemptContext context) { return(new _RecordReader_93(this)); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <Text, Text> CreateRecordReader(InputSplit genericSplit , TaskAttemptContext context) { context.SetStatus(genericSplit.ToString()); return(new KeyValueLineRecordReader(context.GetConfiguration())); }
/// <exception cref="System.IO.IOException"/> public override void Initialize(InputSplit genericSplit, TaskAttemptContext context ) { lineRecordReader.Initialize(genericSplit, context); }
// A reporter that does nothing /// <exception cref="System.Exception"/> public virtual void TestFormat() { JobConf job = new JobConf(defaultConf); Random random = new Random(); long seed = random.NextLong(); Log.Info("seed = " + seed); random.SetSeed(seed); localFs.Delete(workDir, true); FileInputFormat.SetInputPaths(job, workDir); int length = 10000; int numFiles = 10; CreateFiles(length, numFiles, random); // create a combined split for the files CombineTextInputFormat format = new CombineTextInputFormat(); LongWritable key = new LongWritable(); Text value = new Text(); for (int i = 0; i < 3; i++) { int numSplits = random.Next(length / 20) + 1; Log.Info("splitting: requesting = " + numSplits); InputSplit[] splits = format.GetSplits(job, numSplits); Log.Info("splitting: got = " + splits.Length); // we should have a single split as the length is comfortably smaller than // the block size NUnit.Framework.Assert.AreEqual("We got more than one splits!", 1, splits.Length); InputSplit split = splits[0]; NUnit.Framework.Assert.AreEqual("It should be CombineFileSplit", typeof(CombineFileSplit ), split.GetType()); // check the split BitSet bits = new BitSet(length); Log.Debug("split= " + split); RecordReader <LongWritable, Text> reader = format.GetRecordReader(split, job, voidReporter ); try { int count = 0; while (reader.Next(key, value)) { int v = System.Convert.ToInt32(value.ToString()); Log.Debug("read " + v); if (bits.Get(v)) { Log.Warn("conflict with " + v + " at position " + reader.GetPos()); } NUnit.Framework.Assert.IsFalse("Key in multiple partitions.", bits.Get(v)); bits.Set(v); count++; } Log.Info("splits=" + split + " count=" + count); } finally { reader.Close(); } NUnit.Framework.Assert.AreEqual("Some keys in no partition.", length, bits.Cardinality ()); } }
/// <exception cref="System.IO.IOException"/> public override RecordReader <BytesWritable, BytesWritable> GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { return(new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader(job, (FileSplit)split)); }
/// <exception cref="System.IO.IOException"/> private static IList <Text> ReadSplit(InputFormat <LongWritable, Text> format, InputSplit split, JobConf job) { IList <Text> result = new AList <Text>(); RecordReader <LongWritable, Text> reader = format.GetRecordReader(split, job, voidReporter ); LongWritable key = reader.CreateKey(); Text value = reader.CreateValue(); while (reader.Next(key, value)) { result.AddItem(value); value = reader.CreateValue(); } reader.Close(); return(result); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Initialize(InputSplit split, TaskAttemptContext context) { originalRR.Initialize(((TaggedInputSplit)split).GetInputSplit(), context); }
/// <exception cref="System.Exception"/> /// <exception cref="System.IO.IOException"/> public TaskSplitMetaInfo(InputSplit split, long startOffset) : this(new JobSplit.TaskSplitIndex(string.Empty, startOffset), split.GetLocations (), split.GetLength()) { }
/// <exception cref="System.IO.IOException"/> public override RecordReader <BytesWritable, BytesWritable> CreateRecordReader(InputSplit split, TaskAttemptContext context) { return(new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()); }
/// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override RecordReader <LongWritable, T> CreateRecordReader(InputSplit split , TaskAttemptContext context) { return(CreateDBRecordReader((DBInputFormat.DBInputSplit)split, context.GetConfiguration ())); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, Text> GetRecordReader(InputSplit genericSplit , JobConf job, Reporter reporter) { reporter.SetStatus(genericSplit.ToString()); return(new LineRecordReader(job, (FileSplit)genericSplit)); }
/// <exception cref="System.IO.IOException"/> public virtual RecordReader <FloatWritable, NullWritable> GetRecordReader(InputSplit genericSplit, JobConf job, Reporter reporter) { return(new PipesNonJavaInputFormat.PipesDummyRecordReader(job, genericSplit)); }
/// <exception cref="System.IO.IOException"/> public virtual RecordReader <IntWritable, NullWritable> GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { return(new _RecordReader_145(this, split)); }
/// <exception cref="System.IO.IOException"/> public PipesDummyRecordReader(Configuration job, InputSplit split) { }
/// <summary>This is not implemented yet.</summary> /// <exception cref="System.IO.IOException"/> public abstract RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job, Reporter reporter);
/// <exception cref="System.IO.IOException"/> public override RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { return(new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, typeof( CombineFileRecordReader))); }
public virtual RecordReader <K, V> GetRecordReader(InputSplit ignored, JobConf conf , Reporter reporter) { return(new _RecordReader_408(this)); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf conf , Reporter reporter) { return(new CombineFileRecordReader(conf, (CombineFileSplit)split, reporter, typeof( CombineSequenceFileInputFormat.SequenceFileRecordReaderWrapper))); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public abstract override RecordReader <K, V> CreateRecordReader(InputSplit split, TaskAttemptContext context);
public abstract ComposableRecordReader GetRecordReader(InputSplit arg1, JobConf arg2 , Reporter arg3);
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, Text> CreateRecordReader(InputSplit genericSplit , TaskAttemptContext context) { context.SetStatus(genericSplit.ToString()); return(new LineRecordReader()); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, Text> CreateRecordReader(InputSplit split , TaskAttemptContext context) { return(new CombineFileRecordReader <LongWritable, Text>((CombineFileSplit)split, context , typeof(CombineTextInputFormat.TextRecordReaderWrapper))); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Initialize(InputSplit split, TaskAttemptContext context) { base.Initialize(split, context); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <Text, Text> CreateRecordReader(InputSplit split, TaskAttemptContext context) { return(new TeraInputFormat.TeraRecordReader()); }