/// <exception cref="System.IO.IOException"/>
public virtual void RunValueIterator(Path tmpDir, TestReduceTask.Pair[] vals, Configuration
conf, CompressionCodec codec)
{
FileSystem localFs = FileSystem.GetLocal(conf);
FileSystem rfs = ((LocalFileSystem)localFs).GetRaw();
Path path = new Path(tmpDir, "data.in");
IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, rfs.Create(path
), typeof(Text), typeof(Text), codec, null);
foreach (TestReduceTask.Pair p in vals)
{
writer.Append(new Text(p.key), new Text(p.value));
}
writer.Close();
RawKeyValueIterator rawItr = Merger.Merge <Text, Text>(conf, rfs, codec, new Path[]
{ path }, false, conf.GetInt(JobContext.IoSortFactor, 100), tmpDir, new Text.Comparator
(), new TestReduceTask.NullProgress(), null, null, null);
Task.ValuesIterator valItr = new Task.ValuesIterator <Text, Text>(rawItr, WritableComparator
.Get(typeof(Text)), typeof(Text), typeof(Text), conf, new TestReduceTask.NullProgress
());
// WritableComparators are not generic
int i = 0;
while (valItr.More())
{
object key = valItr.GetKey();
string keyString = key.ToString();
// make sure it matches!
NUnit.Framework.Assert.AreEqual(vals[i].key, keyString);
// must have at least 1 value!
NUnit.Framework.Assert.IsTrue(valItr.HasNext());
while (valItr.HasNext())
{
string valueString = valItr.Next().ToString();
// make sure the values match
NUnit.Framework.Assert.AreEqual(vals[i].value, valueString);
// make sure the keys match
NUnit.Framework.Assert.AreEqual(vals[i].key, valItr.GetKey().ToString());
i += 1;
}
// make sure the key hasn't changed under the hood
NUnit.Framework.Assert.AreEqual(keyString, valItr.GetKey().ToString());
valItr.NextKey();
}
NUnit.Framework.Assert.AreEqual(vals.Length, i);
// make sure we have progress equal to 1.0
NUnit.Framework.Assert.AreEqual(1.0f, rawItr.GetProgress().Get());
}
/// <exception cref="System.IO.IOException"/>
public static void WriteFile <K, V>(RawKeyValueIterator records, IFile.Writer <K, V
> writer, Progressable progressable, Configuration conf)
{
long progressBar = conf.GetLong(JobContext.RecordsBeforeProgress, 10000);
long recordCtr = 0;
while (records.Next())
{
writer.Append(records.GetKey(), records.GetValue());
if (((recordCtr++) % progressBar) == 0)
{
progressable.Progress();
}
}
}