/// <exception cref="System.Exception"/>
public virtual void TestNumInputs()
{
JobConf job = new JobConf(conf);
dfs = NewDFSCluster(job);
FileSystem fs = dfs.GetFileSystem();
System.Console.Out.WriteLine("FileSystem " + fs.GetUri());
Path inputDir = new Path("/foo/");
int numFiles = 10;
string fileNameBase = "part-0000";
for (int i = 0; i < numFiles; ++i)
{
CreateInputs(fs, inputDir, fileNameBase + i.ToString());
}
CreateInputs(fs, inputDir, "_meta");
CreateInputs(fs, inputDir, "_temp");
// split it using a file input format
TextInputFormat.AddInputPath(job, inputDir);
TextInputFormat inFormat = new TextInputFormat();
inFormat.Configure(job);
InputSplit[] splits = inFormat.GetSplits(job, 1);
NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles
, numFiles, job.GetLong(FileInputFormat.NumInputFiles, 0));
}
/// <exception cref="System.Exception"/>
public virtual void TestLocality()
{
JobConf job = new JobConf(conf);
dfs = NewDFSCluster(job);
FileSystem fs = dfs.GetFileSystem();
System.Console.Out.WriteLine("FileSystem " + fs.GetUri());
Path inputDir = new Path("/foo/");
string fileName = "part-0000";
CreateInputs(fs, inputDir, fileName);
// split it using a file input format
TextInputFormat.AddInputPath(job, inputDir);
TextInputFormat inFormat = new TextInputFormat();
inFormat.Configure(job);
InputSplit[] splits = inFormat.GetSplits(job, 1);
FileStatus fileStatus = fs.GetFileStatus(new Path(inputDir, fileName));
BlockLocation[] locations = fs.GetFileBlockLocations(fileStatus, 0, fileStatus.GetLen
());
System.Console.Out.WriteLine("Made splits");
// make sure that each split is a block and the locations match
for (int i = 0; i < splits.Length; ++i)
{
FileSplit fileSplit = (FileSplit)splits[i];
System.Console.Out.WriteLine("File split: " + fileSplit);
foreach (string h in fileSplit.GetLocations())
{
System.Console.Out.WriteLine("Location: " + h);
}
System.Console.Out.WriteLine("Block: " + locations[i]);
NUnit.Framework.Assert.AreEqual(locations[i].GetOffset(), fileSplit.GetStart());
NUnit.Framework.Assert.AreEqual(locations[i].GetLength(), fileSplit.GetLength());
string[] blockLocs = locations[i].GetHosts();
string[] splitLocs = fileSplit.GetLocations();
NUnit.Framework.Assert.AreEqual(2, blockLocs.Length);
NUnit.Framework.Assert.AreEqual(2, splitLocs.Length);
NUnit.Framework.Assert.IsTrue((blockLocs[0].Equals(splitLocs[0]) && blockLocs[1].
Equals(splitLocs[1])) || (blockLocs[1].Equals(splitLocs[0]) && blockLocs[0].Equals
(splitLocs[1])));
}
NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles
, 1, job.GetLong(FileInputFormat.NumInputFiles, 0));
}