/// <summary>Combines values for a given key.</summary>
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values.
/// </param>
/// <param name="values">the values to combine</param>
/// <param name="context">to collect combined values</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text val in values)
{
aggregator.AddNextValue(val);
}
IEnumerator <object> outputs = aggregator.GetCombinerOutput().GetEnumerator();
while (outputs.HasNext())
{
object v = outputs.Next();
if (v is Text)
{
context.Write(key, (Text)v);
}
else
{
context.Write(key, new Text(v.ToString()));
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
// Make one reducer slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_reduce = conf.GetBoolean(MRJobConfig.ReduceSpeculative, false
);
// IF TESTING REDUCE SPECULATIVE EXECUTION:
// Make the "*_r_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_r_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Reduce) && test_speculate_reduce && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(key, new IntWritable(0));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
NUnit.Framework.Assert.AreEqual(prop, conf.Get("a"));
WriteFlag(conf, "reduce.setup." + name);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
foreach (Text value in values)
{
WriteFlag(context.GetConfiguration(), "reduce." + name + ".value." + value);
context.Write(key, new Text(value + name));
}
}
/// <summary>Reduce task done, write output to a file.</summary>
/// <exception cref="System.IO.IOException"/>
protected override void Cleanup(Reducer.Context context)
{
//write output to a file
Configuration conf = context.GetConfiguration();
Path outDir = new Path(conf.Get(FileOutputFormat.Outdir));
Path outFile = new Path(outDir, "reduce-out");
FileSystem fileSys = FileSystem.Get(conf);
SequenceFile.Writer writer = SequenceFile.CreateWriter(fileSys, conf, outFile, typeof(
LongWritable), typeof(LongWritable), SequenceFile.CompressionType.None);
writer.Append(new LongWritable(numInside), new LongWritable(numOutside));
writer.Close();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
this.fieldSeparator = conf.Get(FieldSelectionHelper.DataFieldSeperator, "\t");
this.reduceOutputKeyValueSpec = conf.Get(FieldSelectionHelper.ReduceOutputKeyValueSpec
, "0-:");
allReduceValueFieldsFrom = FieldSelectionHelper.ParseOutputKeyValueSpec(reduceOutputKeyValueSpec
, reduceOutputKeyFieldList, reduceOutputValueFieldList);
Log.Info(FieldSelectionHelper.SpecToString(fieldSeparator, reduceOutputKeyValueSpec
, allReduceValueFieldsFrom, reduceOutputKeyFieldList, reduceOutputValueFieldList
));
}
/// <summary>Write output to files.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
Path dir = new Path(conf.Get(WorkingDirProperty));
FileSystem fs = dir.GetFileSystem(conf);
{
// write hex output
Path hexfile = new Path(conf.Get(HexFileProperty));
OutputStream @out = new BufferedOutputStream(fs.Create(hexfile));
try
{
foreach (byte b in hex)
{
@out.Write(b);
}
}
finally
{
@out.Close();
}
}
// If the starting digit is 1,
// the hex value can be converted to decimal value.
if (conf.GetInt(DigitStartProperty, 1) == 1)
{
Path outfile = new Path(dir, "pi.txt");
Log.Info("Writing text output to " + outfile);
OutputStream outputstream = fs.Create(outfile);
try
{
PrintWriter @out = new PrintWriter(new OutputStreamWriter(outputstream, Charsets.
Utf8), true);
// write hex text
Print(@out, hex.GetEnumerator(), "Pi = 0x3.", "%02X", 5, 5);
@out.WriteLine("Total number of hexadecimal digits is " + 2 * hex.Count + ".");
// write decimal text
BaileyBorweinPlouffe.Fraction dec = new BaileyBorweinPlouffe.Fraction(hex);
int decDigits = 2 * hex.Count;
// TODO: this is conservative.
Print(@out, new _IEnumerator_168(decDigits, dec), "Pi = 3.", "%d", 10, 5);
@out.WriteLine("Total number of decimal digits is " + decDigits + ".");
}
finally
{
outputstream.Close();
}
}
}
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values. In effect, data
/// driven computing is achieved. It is assumed that each aggregator's
/// getReport method emits appropriate output for the aggregator. This
/// may be further customized.
/// </param>
/// <param name="values">the values to be aggregated</param>
/// <param name="context"></param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
keyStr = Sharpen.Runtime.Substring(keyStr, pos + ValueAggregatorDescriptor.TypeSeparator
.Length);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text value in values)
{
aggregator.AddNextValue(value);
}
string val = aggregator.GetReport();
key = new Text(keyStr);
context.Write(key, new Text(val));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Reducer.Context context)
{
ValueAggregatorJobBase.Setup(context.GetConfiguration());
}
protected internal override void Setup(Reducer.Context context)
{
chain = new Org.Apache.Hadoop.Mapreduce.Lib.Chain.Chain(false);
chain.Setup(context.GetConfiguration());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
WriteFlag(context.GetConfiguration(), "reduce.cleanup." + name);
}
protected override void Setup(Reducer.Context context)
{
srcs = context.GetConfiguration().GetInt("testdatamerge.sources", 0);
NUnit.Framework.Assert.IsTrue("Invalid src count: " + srcs, srcs > 0);
}