/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Mapper.Context context)
{
// Find the Mapper from the TaggedInputSplit.
TaggedInputSplit inputSplit = (TaggedInputSplit)context.GetInputSplit();
mapper = (Mapper <K1, V1, K2, V2>)ReflectionUtils.NewInstance(inputSplit.GetMapperClass
(), context.GetConfiguration());
}
/// <summary>Constructs the DelegatingRecordReader.</summary>
/// <param name="split">TaggegInputSplit object</param>
/// <param name="context">TaskAttemptContext object</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public DelegatingRecordReader(InputSplit split, TaskAttemptContext context)
{
// Find the InputFormat and then the RecordReader from the
// TaggedInputSplit.
TaggedInputSplit taggedInputSplit = (TaggedInputSplit)split;
InputFormat <K, V> inputFormat = (InputFormat <K, V>)ReflectionUtils.NewInstance(taggedInputSplit
.GetInputFormatClass(), context.GetConfiguration());
originalRR = inputFormat.CreateRecordReader(taggedInputSplit.GetInputSplit(), context
);
}
/// <exception cref="System.Exception"/>
public virtual void TestSplitting()
{
Job job = Job.GetInstance();
MiniDFSCluster dfs = null;
try
{
dfs = new MiniDFSCluster.Builder(job.GetConfiguration()).NumDataNodes(4).Racks(new
string[] { "/rack0", "/rack0", "/rack1", "/rack1" }).Hosts(new string[] { "host0"
, "host1", "host2", "host3" }).Build();
FileSystem fs = dfs.GetFileSystem();
Path path = GetPath("/foo/bar", fs);
Path path2 = GetPath("/foo/baz", fs);
Path path3 = GetPath("/bar/bar", fs);
Path path4 = GetPath("/bar/baz", fs);
int numSplits = 100;
FileInputFormat.SetMaxInputSplitSize(job, fs.GetFileStatus(path).GetLen() / numSplits
);
MultipleInputs.AddInputPath(job, path, typeof(TextInputFormat), typeof(TestDelegatingInputFormat.MapClass
));
MultipleInputs.AddInputPath(job, path2, typeof(TextInputFormat), typeof(TestDelegatingInputFormat.MapClass2
));
MultipleInputs.AddInputPath(job, path3, typeof(KeyValueTextInputFormat), typeof(TestDelegatingInputFormat.MapClass
));
MultipleInputs.AddInputPath(job, path4, typeof(TextInputFormat), typeof(TestDelegatingInputFormat.MapClass2
));
DelegatingInputFormat inFormat = new DelegatingInputFormat();
int[] bins = new int[3];
foreach (InputSplit split in (IList <InputSplit>)inFormat.GetSplits(job))
{
NUnit.Framework.Assert.IsTrue(split is TaggedInputSplit);
TaggedInputSplit tis = (TaggedInputSplit)split;
int index = -1;
if (tis.GetInputFormatClass().Equals(typeof(KeyValueTextInputFormat)))
{
// path3
index = 0;
}
else
{
if (tis.GetMapperClass().Equals(typeof(TestDelegatingInputFormat.MapClass)))
{
// path
index = 1;
}
else
{
// path2 and path4
index = 2;
}
}
bins[index]++;
}
NUnit.Framework.Assert.AreEqual("count is not equal to num splits", numSplits, bins
[0]);
NUnit.Framework.Assert.AreEqual("count is not equal to num splits", numSplits, bins
[1]);
NUnit.Framework.Assert.AreEqual("count is not equal to 2 * num splits", numSplits
* 2, bins[2]);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
}
}