/// <exception cref="System.Exception"/> public virtual void TestNumInputs() { JobConf job = new JobConf(conf); dfs = NewDFSCluster(job); FileSystem fs = dfs.GetFileSystem(); System.Console.Out.WriteLine("FileSystem " + fs.GetUri()); Path inputDir = new Path("/foo/"); int numFiles = 10; string fileNameBase = "part-0000"; for (int i = 0; i < numFiles; ++i) { CreateInputs(fs, inputDir, fileNameBase + i.ToString()); } CreateInputs(fs, inputDir, "_meta"); CreateInputs(fs, inputDir, "_temp"); // split it using a file input format TextInputFormat.AddInputPath(job, inputDir); TextInputFormat inFormat = new TextInputFormat(); inFormat.Configure(job); InputSplit[] splits = inFormat.GetSplits(job, 1); NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles , numFiles, job.GetLong(FileInputFormat.NumInputFiles, 0)); }
/// <exception cref="System.Exception"/> public virtual void TestLocality() { JobConf job = new JobConf(conf); dfs = NewDFSCluster(job); FileSystem fs = dfs.GetFileSystem(); System.Console.Out.WriteLine("FileSystem " + fs.GetUri()); Path inputDir = new Path("/foo/"); string fileName = "part-0000"; CreateInputs(fs, inputDir, fileName); // split it using a file input format TextInputFormat.AddInputPath(job, inputDir); TextInputFormat inFormat = new TextInputFormat(); inFormat.Configure(job); InputSplit[] splits = inFormat.GetSplits(job, 1); FileStatus fileStatus = fs.GetFileStatus(new Path(inputDir, fileName)); BlockLocation[] locations = fs.GetFileBlockLocations(fileStatus, 0, fileStatus.GetLen ()); System.Console.Out.WriteLine("Made splits"); // make sure that each split is a block and the locations match for (int i = 0; i < splits.Length; ++i) { FileSplit fileSplit = (FileSplit)splits[i]; System.Console.Out.WriteLine("File split: " + fileSplit); foreach (string h in fileSplit.GetLocations()) { System.Console.Out.WriteLine("Location: " + h); } System.Console.Out.WriteLine("Block: " + locations[i]); NUnit.Framework.Assert.AreEqual(locations[i].GetOffset(), fileSplit.GetStart()); NUnit.Framework.Assert.AreEqual(locations[i].GetLength(), fileSplit.GetLength()); string[] blockLocs = locations[i].GetHosts(); string[] splitLocs = fileSplit.GetLocations(); NUnit.Framework.Assert.AreEqual(2, blockLocs.Length); NUnit.Framework.Assert.AreEqual(2, splitLocs.Length); NUnit.Framework.Assert.IsTrue((blockLocs[0].Equals(splitLocs[0]) && blockLocs[1]. Equals(splitLocs[1])) || (blockLocs[1].Equals(splitLocs[0]) && blockLocs[0].Equals (splitLocs[1]))); } NUnit.Framework.Assert.AreEqual("Expected value of " + FileInputFormat.NumInputFiles , 1, job.GetLong(FileInputFormat.NumInputFiles, 0)); }