/// <exception cref="System.IO.IOException"/>
public virtual RecordReader <object, object> GetRecordReader(InputSplit split, JobConf
job, Reporter reporter)
{
return(new _RecordReader_62());
}
public override RecordReader <K, V> CreateRecordReader(InputSplit ignored, TaskAttemptContext
context)
{
return(new TestWrappedRRClassloader.RR_ClassLoaderChecker <K, V>(context.GetConfiguration
()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override RecordReader <Text, Text> CreateRecordReader(InputSplit split, TaskAttemptContext
context)
{
return(new RandomTextWriterJob.RandomInputFormat.RandomRecordReader(((FileSplit)split
).GetPath()));
}
// abstract method from super class implemented to return null
/// <exception cref="System.IO.IOException"/>
public override RecordReader <K, V> CreateRecordReader(InputSplit split, TaskAttemptContext
context)
{
return(null);
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter
reporter)
{
return(null);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
this.i.Set(((TestInputSampler.SequentialSplit)split).GetInit() - 1);
this.maxVal = this.i.Get() + this._enclosing.maxDepth + 1;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override RecordReader <IntWritable, NullWritable> CreateRecordReader(InputSplit
split, TaskAttemptContext context)
{
return(new _RecordReader_93(this));
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <Text, Text> CreateRecordReader(InputSplit genericSplit
, TaskAttemptContext context)
{
context.SetStatus(genericSplit.ToString());
return(new KeyValueLineRecordReader(context.GetConfiguration()));
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(InputSplit genericSplit, TaskAttemptContext context
)
{
lineRecordReader.Initialize(genericSplit, context);
}
// A reporter that does nothing
/// <exception cref="System.Exception"/>
public virtual void TestFormat()
{
JobConf job = new JobConf(defaultConf);
Random random = new Random();
long seed = random.NextLong();
Log.Info("seed = " + seed);
random.SetSeed(seed);
localFs.Delete(workDir, true);
FileInputFormat.SetInputPaths(job, workDir);
int length = 10000;
int numFiles = 10;
CreateFiles(length, numFiles, random);
// create a combined split for the files
CombineTextInputFormat format = new CombineTextInputFormat();
LongWritable key = new LongWritable();
Text value = new Text();
for (int i = 0; i < 3; i++)
{
int numSplits = random.Next(length / 20) + 1;
Log.Info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.GetSplits(job, numSplits);
Log.Info("splitting: got = " + splits.Length);
// we should have a single split as the length is comfortably smaller than
// the block size
NUnit.Framework.Assert.AreEqual("We got more than one splits!", 1, splits.Length);
InputSplit split = splits[0];
NUnit.Framework.Assert.AreEqual("It should be CombineFileSplit", typeof(CombineFileSplit
), split.GetType());
// check the split
BitSet bits = new BitSet(length);
Log.Debug("split= " + split);
RecordReader <LongWritable, Text> reader = format.GetRecordReader(split, job, voidReporter
);
try
{
int count = 0;
while (reader.Next(key, value))
{
int v = System.Convert.ToInt32(value.ToString());
Log.Debug("read " + v);
if (bits.Get(v))
{
Log.Warn("conflict with " + v + " at position " + reader.GetPos());
}
NUnit.Framework.Assert.IsFalse("Key in multiple partitions.", bits.Get(v));
bits.Set(v);
count++;
}
Log.Info("splits=" + split + " count=" + count);
}
finally
{
reader.Close();
}
NUnit.Framework.Assert.AreEqual("Some keys in no partition.", length, bits.Cardinality
());
}
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <BytesWritable, BytesWritable> GetRecordReader(InputSplit
split, JobConf job, Reporter reporter)
{
return(new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader(job,
(FileSplit)split));
}
/// <exception cref="System.IO.IOException"/>
private static IList <Text> ReadSplit(InputFormat <LongWritable, Text> format, InputSplit
split, JobConf job)
{
IList <Text> result = new AList <Text>();
RecordReader <LongWritable, Text> reader = format.GetRecordReader(split, job, voidReporter
);
LongWritable key = reader.CreateKey();
Text value = reader.CreateValue();
while (reader.Next(key, value))
{
result.AddItem(value);
value = reader.CreateValue();
}
reader.Close();
return(result);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
originalRR.Initialize(((TaggedInputSplit)split).GetInputSplit(), context);
}
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public TaskSplitMetaInfo(InputSplit split, long startOffset)
: this(new JobSplit.TaskSplitIndex(string.Empty, startOffset), split.GetLocations
(), split.GetLength())
{
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <BytesWritable, BytesWritable> CreateRecordReader(InputSplit
split, TaskAttemptContext context)
{
return(new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader());
}
/// <summary>
/// <inheritDoc/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override RecordReader <LongWritable, T> CreateRecordReader(InputSplit split
, TaskAttemptContext context)
{
return(CreateDBRecordReader((DBInputFormat.DBInputSplit)split, context.GetConfiguration
()));
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <LongWritable, Text> GetRecordReader(InputSplit genericSplit
, JobConf job, Reporter reporter)
{
reporter.SetStatus(genericSplit.ToString());
return(new LineRecordReader(job, (FileSplit)genericSplit));
}
/// <exception cref="System.IO.IOException"/>
public virtual RecordReader <FloatWritable, NullWritable> GetRecordReader(InputSplit
genericSplit, JobConf job, Reporter reporter)
{
return(new PipesNonJavaInputFormat.PipesDummyRecordReader(job, genericSplit));
}
/// <exception cref="System.IO.IOException"/>
public virtual RecordReader <IntWritable, NullWritable> GetRecordReader(InputSplit
split, JobConf job, Reporter reporter)
{
return(new _RecordReader_145(this, split));
}
/// <exception cref="System.IO.IOException"/>
public PipesDummyRecordReader(Configuration job, InputSplit split)
{
}
/// <summary>This is not implemented yet.</summary>
/// <exception cref="System.IO.IOException"/>
public abstract RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
Reporter reporter);
/// <exception cref="System.IO.IOException"/>
public override RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter
reporter)
{
return(new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, typeof(
CombineFileRecordReader)));
}
public virtual RecordReader <K, V> GetRecordReader(InputSplit ignored, JobConf conf
, Reporter reporter)
{
return(new _RecordReader_408(this));
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf conf
, Reporter reporter)
{
return(new CombineFileRecordReader(conf, (CombineFileSplit)split, reporter, typeof(
CombineSequenceFileInputFormat.SequenceFileRecordReaderWrapper)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public abstract override RecordReader <K, V> CreateRecordReader(InputSplit split,
TaskAttemptContext context);
public abstract ComposableRecordReader GetRecordReader(InputSplit arg1, JobConf arg2
, Reporter arg3);
/// <exception cref="System.IO.IOException"/>
public override RecordReader <LongWritable, Text> CreateRecordReader(InputSplit genericSplit
, TaskAttemptContext context)
{
context.SetStatus(genericSplit.ToString());
return(new LineRecordReader());
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <LongWritable, Text> CreateRecordReader(InputSplit split
, TaskAttemptContext context)
{
return(new CombineFileRecordReader <LongWritable, Text>((CombineFileSplit)split, context
, typeof(CombineTextInputFormat.TextRecordReaderWrapper)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
base.Initialize(split, context);
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <Text, Text> CreateRecordReader(InputSplit split, TaskAttemptContext
context)
{
return(new TeraInputFormat.TeraRecordReader());
}
