/// <exception cref="System.IO.IOException"/> public virtual void RunValueIterator(Path tmpDir, TestReduceTask.Pair[] vals, Configuration conf, CompressionCodec codec) { FileSystem localFs = FileSystem.GetLocal(conf); FileSystem rfs = ((LocalFileSystem)localFs).GetRaw(); Path path = new Path(tmpDir, "data.in"); IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, rfs.Create(path ), typeof(Text), typeof(Text), codec, null); foreach (TestReduceTask.Pair p in vals) { writer.Append(new Text(p.key), new Text(p.value)); } writer.Close(); RawKeyValueIterator rawItr = Merger.Merge <Text, Text>(conf, rfs, codec, new Path[] { path }, false, conf.GetInt(JobContext.IoSortFactor, 100), tmpDir, new Text.Comparator (), new TestReduceTask.NullProgress(), null, null, null); Task.ValuesIterator valItr = new Task.ValuesIterator <Text, Text>(rawItr, WritableComparator .Get(typeof(Text)), typeof(Text), typeof(Text), conf, new TestReduceTask.NullProgress ()); // WritableComparators are not generic int i = 0; while (valItr.More()) { object key = valItr.GetKey(); string keyString = key.ToString(); // make sure it matches! NUnit.Framework.Assert.AreEqual(vals[i].key, keyString); // must have at least 1 value! NUnit.Framework.Assert.IsTrue(valItr.HasNext()); while (valItr.HasNext()) { string valueString = valItr.Next().ToString(); // make sure the values match NUnit.Framework.Assert.AreEqual(vals[i].value, valueString); // make sure the keys match NUnit.Framework.Assert.AreEqual(vals[i].key, valItr.GetKey().ToString()); i += 1; } // make sure the key hasn't changed under the hood NUnit.Framework.Assert.AreEqual(keyString, valItr.GetKey().ToString()); valItr.NextKey(); } NUnit.Framework.Assert.AreEqual(vals.Length, i); // make sure we have progress equal to 1.0 NUnit.Framework.Assert.AreEqual(1.0f, rawItr.GetProgress().Get()); }
/// <exception cref="System.IO.IOException"/> public static void WriteFile <K, V>(RawKeyValueIterator records, IFile.Writer <K, V > writer, Progressable progressable, Configuration conf) { long progressBar = conf.GetLong(JobContext.RecordsBeforeProgress, 10000); long recordCtr = 0; while (records.Next()) { writer.Append(records.GetKey(), records.GetValue()); if (((recordCtr++) % progressBar) == 0) { progressable.Progress(); } } }