/// <exception cref="System.Exception"/>
public virtual void TestFormatWithLessPathsThanSplits()
{
MultiFileInputFormat <Text, Text> format = new TestMultiFileInputFormat.DummyMultiFileInputFormat
(this);
FileSystem fs = FileSystem.GetLocal(job);
// Test with no path
InitFiles(fs, 0, -1);
NUnit.Framework.Assert.AreEqual(0, format.GetSplits(job, 2).Length);
// Test with 2 path and 4 splits
InitFiles(fs, 2, 500);
NUnit.Framework.Assert.AreEqual(2, format.GetSplits(job, 4).Length);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFormat()
{
Log.Info("Test started");
Log.Info("Max split count = " + MaxSplitCount);
Log.Info("Split count increment = " + SplitCountIncr);
Log.Info("Max bytes per file = " + MaxBytes);
Log.Info("Max number of files = " + MaxNumFiles);
Log.Info("Number of files increment = " + NumFilesIncr);
MultiFileInputFormat <Text, Text> format = new TestMultiFileInputFormat.DummyMultiFileInputFormat
(this);
FileSystem fs = FileSystem.GetLocal(job);
for (int numFiles = 1; numFiles < MaxNumFiles; numFiles += (NumFilesIncr / 2) + rand
.Next(NumFilesIncr / 2))
{
Path dir = InitFiles(fs, numFiles, -1);
BitSet bits = new BitSet(numFiles);
for (int i = 1; i < MaxSplitCount; i += rand.Next(SplitCountIncr) + 1)
{
Log.Info("Running for Num Files=" + numFiles + ", split count=" + i);
MultiFileSplit[] splits = (MultiFileSplit[])format.GetSplits(job, i);
bits.Clear();
foreach (MultiFileSplit split in splits)
{
long splitLength = 0;
foreach (Path p in split.GetPaths())
{
long length = fs.GetContentSummary(p).GetLength();
NUnit.Framework.Assert.AreEqual(length, lengths[p.GetName()]);
splitLength += length;
string name = p.GetName();
int index = System.Convert.ToInt32(Sharpen.Runtime.Substring(name, name.LastIndexOf
("file_") + 5));
NUnit.Framework.Assert.IsFalse(bits.Get(index));
bits.Set(index);
}
NUnit.Framework.Assert.AreEqual(splitLength, split.GetLength());
}
}
NUnit.Framework.Assert.AreEqual(bits.Cardinality(), numFiles);
fs.Delete(dir, true);
}
Log.Info("Test Finished");
}