/// <summary>Combines values for a given key.</summary>
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values.
/// </param>
/// <param name="values">the values to combine</param>
/// <param name="context">to collect combined values</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text val in values)
{
aggregator.AddNextValue(val);
}
IEnumerator <object> outputs = aggregator.GetCombinerOutput().GetEnumerator();
while (outputs.HasNext())
{
object v = outputs.Next();
if (v is Text)
{
context.Write(key, (Text)v);
}
else
{
context.Write(key, new Text(v.ToString()));
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
NUnit.Framework.Assert.AreEqual(prop, conf.Get("a"));
WriteFlag(conf, "reduce.setup." + name);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private void RunNewReducer <Inkey, Invalue, Outkey, Outvalue>(JobConf job, TaskUmbilicalProtocol
umbilical, Task.TaskReporter reporter, RawKeyValueIterator rIter, RawComparator
<INKEY> comparator)
{
System.Type keyClass = typeof(INKEY);
System.Type valueClass = typeof(INVALUE);
// wrap value iterator to report progress.
RawKeyValueIterator rawIter = rIter;
rIter = new _RawKeyValueIterator_587(rawIter, reporter);
// make a task context so we can get the classes
TaskAttemptContext taskContext = new TaskAttemptContextImpl(job, GetTaskID(), reporter
);
// make a reducer
Reducer <INKEY, INVALUE, OUTKEY, OUTVALUE> reducer = (Reducer <INKEY, INVALUE, OUTKEY
, OUTVALUE>)ReflectionUtils.NewInstance(taskContext.GetReducerClass(), job);
RecordWriter <OUTKEY, OUTVALUE> trackedRW = new ReduceTask.NewTrackingRecordWriter
<OUTKEY, OUTVALUE>(this, taskContext);
job.SetBoolean("mapred.skip.on", IsSkipping());
job.SetBoolean(JobContext.SkipRecords, IsSkipping());
Reducer.Context reducerContext = CreateReduceContext(reducer, job, GetTaskID(), rIter
, reduceInputKeyCounter, reduceInputValueCounter, trackedRW, committer, reporter
, comparator, keyClass, valueClass);
try
{
reducer.Run(reducerContext);
}
finally
{
trackedRW.Close(reducerContext);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal ReduceRunner(Chain _enclosing, Reducer.Context context, Reducer <KEYIN, VALUEIN
, KEYOUT, VALUEOUT> reducer, RecordWriter <KEYOUT, VALUEOUT> rw)
{
this._enclosing = _enclosing;
this.reducer = reducer;
this.chainContext = context;
this.rw = rw;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable index, IEnumerable <SummationWritable>
sums, Reducer.Context context)
{
Log.Info("index=" + index);
foreach (SummationWritable sigma in sums)
{
Compute(sigma.GetElement(), context);
}
}
// should receive 2 rows for each key
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
ctx)
{
count = 0;
foreach (Text value in values)
{
count++;
}
ctx.Write(NullWritable.Get(), new Text(key.ToString() + " " + count));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(NullWritable key, IEnumerable <Unsigned16> values,
Reducer.Context context)
{
Unsigned16 sum = new Unsigned16();
foreach (Unsigned16 val in values)
{
sum.Add(val);
}
context.Write(key, sum);
}
// Run the reducer directly.
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal virtual void RunReducer <Keyin, Valuein, Keyout, Valueout>(TaskInputOutputContext
<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context)
{
RecordWriter <KEYOUT, VALUEOUT> rw = new Chain.ChainRecordWriter <KEYOUT, VALUEOUT>
(context);
Reducer.Context reducerContext = CreateReduceContext(rw, (ReduceContext)context,
rConf);
reducer.Run(reducerContext);
rw.Close(context);
}
/// <summary>
/// Sums all the individual values within the iterator and writes them to the
/// same key.
/// </summary>
/// <param name="key">This will be a length of a word that was read.</param>
/// <param name="values">
/// This will be an iterator of all the values associated with that
/// key.
/// </param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> values,
Reducer.Context context)
{
int sum = 0;
foreach (IntWritable value in values)
{
sum += value.Get();
}
val.Set(sum);
context.Write(key, val);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(KEY key, IEnumerable <LongWritable> values
, Reducer.Context context)
{
long sum = 0;
foreach (LongWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> values,
Reducer.Context context)
{
int seen = 0;
foreach (IntWritable value in values)
{
seen += value.Get();
}
NUnit.Framework.Assert.IsTrue("Bad count for " + key.Get(), Verify(key.Get(), seen
));
context.Write(key, new IntWritable(seen));
}
/// <summary>Reduce task done, write output to a file.</summary>
/// <exception cref="System.IO.IOException"/>
protected override void Cleanup(Reducer.Context context)
{
//write output to a file
Configuration conf = context.GetConfiguration();
Path outDir = new Path(conf.Get(FileOutputFormat.Outdir));
Path outFile = new Path(outDir, "reduce-out");
FileSystem fileSys = FileSystem.Get(conf);
SequenceFile.Writer writer = SequenceFile.CreateWriter(fileSys, conf, outFile, typeof(
LongWritable), typeof(LongWritable), SequenceFile.CompressionType.None);
writer.Append(new LongWritable(numInside), new LongWritable(numOutside));
writer.Close();
}
/// <summary>Concatenate map outputs.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable offset, IEnumerable <BytesWritable> values
, Reducer.Context context)
{
// read map outputs
foreach (BytesWritable bytes in values)
{
for (int i = 0; i < bytes.GetLength(); i++)
{
hex.AddItem(bytes.GetBytes()[i]);
}
}
Log.Info("hex.size() = " + hex.Count);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
this.fieldSeparator = conf.Get(FieldSelectionHelper.DataFieldSeperator, "\t");
this.reduceOutputKeyValueSpec = conf.Get(FieldSelectionHelper.ReduceOutputKeyValueSpec
, "0-:");
allReduceValueFieldsFrom = FieldSelectionHelper.ParseOutputKeyValueSpec(reduceOutputKeyValueSpec
, reduceOutputKeyFieldList, reduceOutputValueFieldList);
Log.Info(FieldSelectionHelper.SpecToString(fieldSeparator, reduceOutputKeyValueSpec
, allReduceValueFieldsFrom, reduceOutputKeyFieldList, reduceOutputValueFieldList
));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString() + this.fieldSeparator;
foreach (Text val in values)
{
FieldSelectionHelper helper = new FieldSelectionHelper();
helper.ExtractOutputKeyValue(keyStr, val.ToString(), fieldSeparator, reduceOutputKeyFieldList
, reduceOutputValueFieldList, allReduceValueFieldsFrom, false, false);
context.Write(helper.GetKey(), helper.GetValue());
}
}
/// <summary>Add reducer that reads from context and writes to a queue</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal virtual void AddReducer(TaskInputOutputContext inputContext, Chain.ChainBlockingQueue
<Chain.KeyValuePair <object, object> > outputQueue)
{
Type keyOutClass = rConf.GetClass(ReducerOutputKeyClass, typeof(object));
Type valueOutClass = rConf.GetClass(ReducerOutputValueClass, typeof(object));
RecordWriter rw = new Chain.ChainRecordWriter(keyOutClass, valueOutClass, outputQueue
, rConf);
Reducer.Context reducerContext = CreateReduceContext(rw, (ReduceContext)inputContext
, rConf);
Chain.ReduceRunner runner = new Chain.ReduceRunner(this, reducerContext, reducer,
rw);
threads.AddItem(runner);
}
/// <summary>Write output to files.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
Path dir = new Path(conf.Get(WorkingDirProperty));
FileSystem fs = dir.GetFileSystem(conf);
{
// write hex output
Path hexfile = new Path(conf.Get(HexFileProperty));
OutputStream @out = new BufferedOutputStream(fs.Create(hexfile));
try
{
foreach (byte b in hex)
{
@out.Write(b);
}
}
finally
{
@out.Close();
}
}
// If the starting digit is 1,
// the hex value can be converted to decimal value.
if (conf.GetInt(DigitStartProperty, 1) == 1)
{
Path outfile = new Path(dir, "pi.txt");
Log.Info("Writing text output to " + outfile);
OutputStream outputstream = fs.Create(outfile);
try
{
PrintWriter @out = new PrintWriter(new OutputStreamWriter(outputstream, Charsets.
Utf8), true);
// write hex text
Print(@out, hex.GetEnumerator(), "Pi = 0x3.", "%02X", 5, 5);
@out.WriteLine("Total number of hexadecimal digits is " + 2 * hex.Count + ".");
// write decimal text
BaileyBorweinPlouffe.Fraction dec = new BaileyBorweinPlouffe.Fraction(hex);
int decDigits = 2 * hex.Count;
// TODO: this is conservative.
Print(@out, new _IEnumerator_168(decDigits, dec), "Pi = 3.", "%d", 10, 5);
@out.WriteLine("Total number of decimal digits is " + decDigits + ".");
}
finally
{
outputstream.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(long key, IEnumerable <string> values, Reducer.Context
context)
{
foreach (string value in values)
{
mos.Write(key, value, value.ToString());
if (!value.ToString().Equals("b"))
{
context.Write(key, value);
}
else
{
mos.Write(Text, key, new Text(Text));
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
if (Error.Equals(key))
{
foreach (Text val in values)
{
context.Write(key, val);
}
}
else
{
if (Checksum.Equals(key))
{
Unsigned16 tmp = new Unsigned16();
Unsigned16 sum = new Unsigned16();
foreach (Text val in values)
{
tmp.Set(val.ToString());
sum.Add(tmp);
}
context.Write(Checksum, new Text(sum.ToString()));
}
else
{
Text value = values.GetEnumerator().Next();
if (firstKey)
{
firstKey = false;
}
else
{
if (value.CompareTo(lastValue) < 0)
{
context.Write(Error, new Text("bad key partitioning:\n file " + lastKey + " key "
+ TextifyBytes(lastValue) + "\n file " + key + " key " + TextifyBytes(value)));
}
}
lastKey.Set(key);
lastValue.Set(value);
}
}
}
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values. In effect, data
/// driven computing is achieved. It is assumed that each aggregator's
/// getReport method emits appropriate output for the aggregator. This
/// may be further customized.
/// </param>
/// <param name="values">the values to be aggregated</param>
/// <param name="context"></param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
keyStr = Sharpen.Runtime.Substring(keyStr, pos + ValueAggregatorDescriptor.TypeSeparator
.Length);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text value in values)
{
aggregator.AddNextValue(value);
}
string val = aggregator.GetReport();
key = new Text(keyStr);
context.Write(key, new Text(val));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Run(Reducer.Context context)
{
Setup(context);
// if no reducer is set, just do nothing
if (chain.GetReducer() == null)
{
return;
}
int numMappers = chain.GetAllMappers().Count;
// if there are no mappers in chain, run the reducer
if (numMappers == 0)
{
chain.RunReducer(context);
return;
}
// add reducer and all mappers with proper context
Chain.ChainBlockingQueue <Chain.KeyValuePair <object, object> > inputqueue;
Chain.ChainBlockingQueue <Chain.KeyValuePair <object, object> > outputqueue;
// add reducer
outputqueue = chain.CreateBlockingQueue();
chain.AddReducer(context, outputqueue);
// add all mappers except last one
for (int i = 0; i < numMappers - 1; i++)
{
inputqueue = outputqueue;
outputqueue = chain.CreateBlockingQueue();
chain.AddMapper(inputqueue, outputqueue, context, i);
}
// add last mapper
chain.AddMapper(outputqueue, context, numMappers - 1);
// start all threads
chain.StartAllThreads();
// wait for all threads
chain.JoinAllThreads();
}
/// <summary>
/// Sums all the individual values within the iterator and writes them to the
/// same key.
/// </summary>
/// <param name="key">This will be one of 2 constants: LENGTH_STR or COUNT_STR.</param>
/// <param name="values">
/// This will be an iterator of all the values associated with that
/// key.
/// </param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <LongWritable> values, Reducer.Context
context)
{
int theSum = 0;
foreach (LongWritable val in values)
{
theSum += val.Get();
}
sum.Set(theSum);
context.Write(key, sum);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Org.Apache.Hadoop.IO.Text key, IEnumerable <IntWritable
> values, Reducer.Context context)
{
int sum = 0;
foreach (IntWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Reducer.Context context)
{
ValueAggregatorJobBase.Setup(context.GetConfiguration());
}
// this reduce throws IOEexception for any input
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
throw new IOException();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
foreach (Text value in values)
{
mos.Write(key, value, value.ToString());
if (!value.ToString().Equals("b"))
{
context.Write(key, value);
}
else
{
mos.Write(Text, key, new Text(Text));
mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B"));
mos.Write(Sequence, new IntWritable(3), new Text(Sequence), (Sequence + "_C"));
}
}
}
/// <exception cref="System.IO.IOException"/>
protected override void Setup(Reducer.Context context)
{
new TestMRWithDistributedCache.DistributedCacheChecker().Setup(context);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
context.GetCounter("MyCounterGroup", "REDUCE_INPUT_GROUPS").Increment(1);
int sum = 0;
foreach (IntWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
context.GetCounter("MyCounterGroup", "REDUCE_OUTPUT_RECORDS").Increment(1);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
mos.Close();
}
protected override void Setup(Reducer.Context context)
{
mos = new MultipleOutputs <long, string>(context);
}
