/// <summary>Sets the Reducer class to the chain job's JobConf.</summary> /// <remarks> /// Sets the Reducer class to the chain job's JobConf. /// <p> /// It has to be specified how key and values are passed from one element of /// the chain to the next, by value or by reference. If a Reducer leverages the /// assumed semantics that the key and values are not modified by the collector /// 'by value' must be used. If the Reducer does not expect this semantics, as /// an optimization to avoid serialization and deserialization 'by reference' /// can be used. /// <p> /// For the added Reducer the configuration given for it, /// <code>reducerConf</code>, have precedence over the job's JobConf. This /// precedence is in effect when the task is running. /// <p> /// IMPORTANT: There is no need to specify the output key/value classes for the /// ChainReducer, this is done by the setReducer or the addMapper for the last /// element in the chain. /// </remarks> /// <param name="job">job's JobConf to add the Reducer class.</param> /// <param name="klass">the Reducer class to add.</param> /// <param name="inputKeyClass">reducer input key class.</param> /// <param name="inputValueClass">reducer input value class.</param> /// <param name="outputKeyClass">reducer output key class.</param> /// <param name="outputValueClass">reducer output value class.</param> /// <param name="byValue"> /// indicates if key/values should be passed by value /// to the next Mapper in the chain, if any. /// </param> /// <param name="reducerConf"> /// a JobConf with the configuration for the Reducer /// class. It is recommended to use a JobConf without default values using the /// <code>JobConf(boolean loadDefaults)</code> constructor with FALSE. /// </param> public static void SetReducer <K1, V1, K2, V2>(JobConf job, Type klass, Type inputKeyClass , Type inputValueClass, Type outputKeyClass, Type outputValueClass, bool byValue , JobConf reducerConf) { job.SetReducerClass(typeof(Org.Apache.Hadoop.Mapred.Lib.ChainReducer)); job.SetOutputKeyClass(outputKeyClass); job.SetOutputValueClass(outputValueClass); Chain.SetReducer(job, klass, inputKeyClass, inputValueClass, outputKeyClass, outputValueClass , byValue, reducerConf); }
public virtual void TestSetReducerWithReducerByValueAsFalse() { JobConf jobConf = new JobConf(); JobConf reducerConf = new JobConf(); Chain.SetReducer(jobConf, typeof(TestChain.MyReducer), typeof(object), typeof(object ), typeof(object), typeof(object), false, reducerConf); bool reduceByValue = reducerConf.GetBoolean("chain.reducer.byValue", true); NUnit.Framework.Assert.AreEqual("It should set chain.reducer.byValue as false " + "in reducerConf when we give value as false", false, reduceByValue); }