public virtual void TestNumInputFilesWithoutRecursively()
{
Configuration conf = GetConfiguration();
conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads);
Job job = Job.GetInstance(conf);
FileInputFormat <object, object> fileInputFormat = new TextInputFormat();
IList <InputSplit> splits = fileInputFormat.GetSplits(job);
NUnit.Framework.Assert.AreEqual("Input splits are not correct", 2, splits.Count);
VerifySplits(Lists.NewArrayList("test:/a1/a2", "test:/a1/file1"), splits);
}
public virtual void TestListStatusNestedNonRecursive()
{
Configuration conf = new Configuration();
conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads);
IList <Path> expectedPaths = ConfigureTestNestedNonRecursive(conf, localFs);
Job job = Job.GetInstance(conf);
FileInputFormat <object, object> fif = new TextInputFormat();
IList <FileStatus> statuses = fif.ListStatus(job);
VerifyFileStatuses(expectedPaths, statuses, localFs);
}
public virtual void TestNumInputFiles()
{
Configuration conf = Org.Mockito.Mockito.Spy(new Configuration());
Job job = MockitoMaker.Make(MockitoMaker.Stub <Job>().Returning(conf).from.GetConfiguration
());
FileStatus stat = MockitoMaker.Make(MockitoMaker.Stub <FileStatus>().Returning(0L)
.from.GetLen());
TextInputFormat ispy = Org.Mockito.Mockito.Spy(new TextInputFormat());
Org.Mockito.Mockito.DoReturn(Arrays.AsList(stat)).When(ispy).ListStatus(job);
ispy.GetSplits(job);
Org.Mockito.Mockito.Verify(conf).SetLong(FileInputFormat.NumInputFiles, 1);
}
public virtual void TestListLocatedStatus()
{
Configuration conf = GetConfiguration();
conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads);
conf.SetBoolean("fs.test.impl.disable.cache", false);
conf.Set(FileInputFormat.InputDir, "test:///a1/a2");
TestFileInputFormat.MockFileSystem mockFs = (TestFileInputFormat.MockFileSystem) new
Path("test:///").GetFileSystem(conf);
NUnit.Framework.Assert.AreEqual("listLocatedStatus already called", 0, mockFs.numListLocatedStatusCalls
);
Job job = Job.GetInstance(conf);
FileInputFormat <object, object> fileInputFormat = new TextInputFormat();
IList <InputSplit> splits = fileInputFormat.GetSplits(job);
NUnit.Framework.Assert.AreEqual("Input splits are not correct", 2, splits.Count);
NUnit.Framework.Assert.AreEqual("listLocatedStatuss calls", 1, mockFs.numListLocatedStatusCalls
);
FileSystem.CloseAll();
}
public virtual void TestNumInputFilesRecursively()
{
Configuration conf = GetConfiguration();
conf.Set(FileInputFormat.InputDirRecursive, "true");
conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads);
Job job = Job.GetInstance(conf);
FileInputFormat <object, object> fileInputFormat = new TextInputFormat();
IList <InputSplit> splits = fileInputFormat.GetSplits(job);
NUnit.Framework.Assert.AreEqual("Input splits are not correct", 3, splits.Count);
VerifySplits(Lists.NewArrayList("test:/a1/a2/file2", "test:/a1/a2/file3", "test:/a1/file1"
), splits);
// Using the deprecated configuration
conf = GetConfiguration();
conf.Set("mapred.input.dir.recursive", "true");
job = Job.GetInstance(conf);
splits = fileInputFormat.GetSplits(job);
VerifySplits(Lists.NewArrayList("test:/a1/a2/file2", "test:/a1/a2/file3", "test:/a1/file1"
), splits);
}
public virtual void TestListStatusErrorOnNonExistantDir()
{
Configuration conf = new Configuration();
conf.SetInt(FileInputFormat.ListStatusNumThreads, numThreads);
ConfigureTestErrorOnNonExistantDir(conf, localFs);
Job job = Job.GetInstance(conf);
FileInputFormat <object, object> fif = new TextInputFormat();
try
{
fif.ListStatus(job);
NUnit.Framework.Assert.Fail("Expecting an IOException for a missing Input path");
}
catch (IOException e)
{
Path expectedExceptionPath = new Path(TestRootDir, "input2");
expectedExceptionPath = localFs.MakeQualified(expectedExceptionPath);
NUnit.Framework.Assert.IsTrue(e is InvalidInputException);
NUnit.Framework.Assert.AreEqual("Input path does not exist: " + expectedExceptionPath
.ToString(), e.Message);
}
}
public virtual void TestSplitLocationInfo()
{
Configuration conf = GetConfiguration();
conf.Set(FileInputFormat.InputDir, "test:///a1/a2");
Job job = Job.GetInstance(conf);
TextInputFormat fileInputFormat = new TextInputFormat();
IList <InputSplit> splits = fileInputFormat.GetSplits(job);
string[] locations = splits[0].GetLocations();
NUnit.Framework.Assert.AreEqual(2, locations.Length);
SplitLocationInfo[] locationInfo = splits[0].GetLocationInfo();
NUnit.Framework.Assert.AreEqual(2, locationInfo.Length);
SplitLocationInfo localhostInfo = locations[0].Equals("localhost") ? locationInfo
[0] : locationInfo[1];
SplitLocationInfo otherhostInfo = locations[0].Equals("otherhost") ? locationInfo
[0] : locationInfo[1];
NUnit.Framework.Assert.IsTrue(localhostInfo.IsOnDisk());
NUnit.Framework.Assert.IsTrue(localhostInfo.IsInMemory());
NUnit.Framework.Assert.IsTrue(otherhostInfo.IsOnDisk());
NUnit.Framework.Assert.IsFalse(otherhostInfo.IsInMemory());
}
