public virtual void TestNumInputFilesRecursively() { Configuration conf = GetConfiguration(); conf.Set(FileInputFormat.InputDirRecursive, "true"); conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads); Job job = Job.GetInstance(conf); FileInputFormat <object, object> fileInputFormat = new TextInputFormat(); IList <InputSplit> splits = fileInputFormat.GetSplits(job); NUnit.Framework.Assert.AreEqual("Input splits are not correct", 3, splits.Count); VerifySplits(Lists.NewArrayList("test:/a1/a2/file2", "test:/a1/a2/file3", "test:/a1/file1" ), splits); // Using the deprecated configuration conf = GetConfiguration(); conf.Set("mapred.input.dir.recursive", "true"); job = Job.GetInstance(conf); splits = fileInputFormat.GetSplits(job); VerifySplits(Lists.NewArrayList("test:/a1/a2/file2", "test:/a1/a2/file3", "test:/a1/file1" ), splits); }
public virtual void TestNumInputFilesWithoutRecursively() { Configuration conf = GetConfiguration(); conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads); Job job = Job.GetInstance(conf); FileInputFormat <object, object> fileInputFormat = new TextInputFormat(); IList <InputSplit> splits = fileInputFormat.GetSplits(job); NUnit.Framework.Assert.AreEqual("Input splits are not correct", 2, splits.Count); VerifySplits(Lists.NewArrayList("test:/a1/a2", "test:/a1/file1"), splits); }
public virtual void TestNumInputFiles() { Configuration conf = Org.Mockito.Mockito.Spy(new Configuration()); Job job = MockitoMaker.Make(MockitoMaker.Stub <Job>().Returning(conf).from.GetConfiguration ()); FileStatus stat = MockitoMaker.Make(MockitoMaker.Stub <FileStatus>().Returning(0L) .from.GetLen()); TextInputFormat ispy = Org.Mockito.Mockito.Spy(new TextInputFormat()); Org.Mockito.Mockito.DoReturn(Arrays.AsList(stat)).When(ispy).ListStatus(job); ispy.GetSplits(job); Org.Mockito.Mockito.Verify(conf).SetLong(FileInputFormat.NumInputFiles, 1); }
public virtual void TestListLocatedStatus() { Configuration conf = GetConfiguration(); conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads); conf.SetBoolean("fs.test.impl.disable.cache", false); conf.Set(FileInputFormat.InputDir, "test:///a1/a2"); TestFileInputFormat.MockFileSystem mockFs = (TestFileInputFormat.MockFileSystem) new Path("test:///").GetFileSystem(conf); NUnit.Framework.Assert.AreEqual("listLocatedStatus already called", 0, mockFs.numListLocatedStatusCalls ); Job job = Job.GetInstance(conf); FileInputFormat <object, object> fileInputFormat = new TextInputFormat(); IList <InputSplit> splits = fileInputFormat.GetSplits(job); NUnit.Framework.Assert.AreEqual("Input splits are not correct", 2, splits.Count); NUnit.Framework.Assert.AreEqual("listLocatedStatuss calls", 1, mockFs.numListLocatedStatusCalls ); FileSystem.CloseAll(); }
public virtual void TestSplitLocationInfo() { Configuration conf = GetConfiguration(); conf.Set(FileInputFormat.InputDir, "test:///a1/a2"); Job job = Job.GetInstance(conf); TextInputFormat fileInputFormat = new TextInputFormat(); IList <InputSplit> splits = fileInputFormat.GetSplits(job); string[] locations = splits[0].GetLocations(); NUnit.Framework.Assert.AreEqual(2, locations.Length); SplitLocationInfo[] locationInfo = splits[0].GetLocationInfo(); NUnit.Framework.Assert.AreEqual(2, locationInfo.Length); SplitLocationInfo localhostInfo = locations[0].Equals("localhost") ? locationInfo [0] : locationInfo[1]; SplitLocationInfo otherhostInfo = locations[0].Equals("otherhost") ? locationInfo [0] : locationInfo[1]; NUnit.Framework.Assert.IsTrue(localhostInfo.IsOnDisk()); NUnit.Framework.Assert.IsTrue(localhostInfo.IsInMemory()); NUnit.Framework.Assert.IsTrue(otherhostInfo.IsOnDisk()); NUnit.Framework.Assert.IsFalse(otherhostInfo.IsInMemory()); }