internal ChainRecordWriter(Type keyClass, Type valueClass, Chain.ChainBlockingQueue <Chain.KeyValuePair <KEYOUT, VALUEOUT> > output, Configuration conf) { // constructor to write to blocking queue this.keyClass = keyClass; this.valueClass = valueClass; this.outputQueue = output; this.conf = conf; }
internal ChainRecordReader(Type keyClass, Type valueClass, Chain.ChainBlockingQueue <Chain.KeyValuePair <KEYIN, VALUEIN> > inputQueue, Configuration conf) { // ChainRecordReader either reads from blocking queue or task context. // constructor to read from a blocking queue this.keyClass = keyClass; this.valueClass = valueClass; this.inputQueue = inputQueue; this.conf = conf; }
/// <summary> /// Add mapper(the last mapper) that reads input from /// queue and writes output to the output context /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal virtual void AddMapper(Chain.ChainBlockingQueue <Chain.KeyValuePair <object , object> > input, TaskInputOutputContext outputContext, int index) { Configuration conf = GetConf(index); Type keyClass = conf.GetClass(MapperInputKeyClass, typeof(object)); Type valueClass = conf.GetClass(MapperInputValueClass, typeof(object)); RecordReader rr = new Chain.ChainRecordReader(keyClass, valueClass, input, conf); RecordWriter rw = new Chain.ChainRecordWriter(outputContext); Chain.MapRunner runner = new Chain.MapRunner(this, mappers[index], CreateMapContext (rr, rw, outputContext, GetConf(index)), rr, rw); threads.AddItem(runner); }
/// <summary>Add reducer that reads from context and writes to a queue</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal virtual void AddReducer(TaskInputOutputContext inputContext, Chain.ChainBlockingQueue <Chain.KeyValuePair <object, object> > outputQueue) { Type keyOutClass = rConf.GetClass(ReducerOutputKeyClass, typeof(object)); Type valueOutClass = rConf.GetClass(ReducerOutputValueClass, typeof(object)); RecordWriter rw = new Chain.ChainRecordWriter(keyOutClass, valueOutClass, outputQueue , rConf); Reducer.Context reducerContext = CreateReduceContext(rw, (ReduceContext)inputContext , rConf); Chain.ReduceRunner runner = new Chain.ReduceRunner(this, reducerContext, reducer, rw); threads.AddItem(runner); }