/// <summary>Combines values for a given key.</summary>
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values.
/// </param>
/// <param name="values">the values to combine</param>
/// <param name="context">to collect combined values</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text val in values)
{
aggregator.AddNextValue(val);
}
IEnumerator <object> outputs = aggregator.GetCombinerOutput().GetEnumerator();
while (outputs.HasNext())
{
object v = outputs.Next();
if (v is Text)
{
context.Write(key, (Text)v);
}
else
{
context.Write(key, new Text(v.ToString()));
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(SecondarySort.IntPair key, IEnumerable <IntWritable
> values, Reducer.Context context)
{
context.Write(Separator, null);
first.Set(Sharpen.Extensions.ToString(key.GetFirst()));
foreach (IntWritable value in values)
{
context.Write(first, value);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
// Make one reducer slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_reduce = conf.GetBoolean(MRJobConfig.ReduceSpeculative, false
);
// IF TESTING REDUCE SPECULATIVE EXECUTION:
// Make the "*_r_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_r_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Reduce) && test_speculate_reduce && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(key, new IntWritable(0));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
foreach (Text value in values)
{
WriteFlag(context.GetConfiguration(), "reduce." + name + ".value." + value);
context.Write(key, new Text(value + name));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
if (Error.Equals(key))
{
foreach (Text val in values)
{
context.Write(key, val);
}
}
else
{
if (Checksum.Equals(key))
{
Unsigned16 tmp = new Unsigned16();
Unsigned16 sum = new Unsigned16();
foreach (Text val in values)
{
tmp.Set(val.ToString());
sum.Add(tmp);
}
context.Write(Checksum, new Text(sum.ToString()));
}
else
{
Text value = values.GetEnumerator().Next();
if (firstKey)
{
firstKey = false;
}
else
{
if (value.CompareTo(lastValue) < 0)
{
context.Write(Error, new Text("bad key partitioning:\n file " + lastKey + " key "
+ TextifyBytes(lastValue) + "\n file " + key + " key " + TextifyBytes(value)));
}
}
lastKey.Set(key);
lastValue.Set(value);
}
}
}
// should receive 2 rows for each key
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
ctx)
{
count = 0;
foreach (Text value in values)
{
count++;
}
ctx.Write(NullWritable.Get(), new Text(key.ToString() + " " + count));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(NullWritable key, IEnumerable <Unsigned16> values,
Reducer.Context context)
{
Unsigned16 sum = new Unsigned16();
foreach (Unsigned16 val in values)
{
sum.Add(val);
}
context.Write(key, sum);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <LongWritable> values, Reducer.Context
context)
{
long sum = 0L;
foreach (LongWritable value in values)
{
sum += value.Get();
}
context.Write(new DBCountPageView.PageviewRecord(key.ToString(), sum), n);
}
/// <summary>
/// Sums all the individual values within the iterator and writes them to the
/// same key.
/// </summary>
/// <param name="key">This will be a length of a word that was read.</param>
/// <param name="values">
/// This will be an iterator of all the values associated with that
/// key.
/// </param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> values,
Reducer.Context context)
{
int sum = 0;
foreach (IntWritable value in values)
{
sum += value.Get();
}
val.Set(sum);
context.Write(key, val);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Org.Apache.Hadoop.IO.Text key, IEnumerable <IntWritable
> values, Reducer.Context context)
{
int sum = 0;
foreach (IntWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(KEY key, IEnumerable <LongWritable> values
, Reducer.Context context)
{
long sum = 0;
foreach (LongWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
}
/// <summary>
/// Sums all the individual values within the iterator and writes them to the
/// same key.
/// </summary>
/// <param name="key">This will be one of 2 constants: LENGTH_STR or COUNT_STR.</param>
/// <param name="values">
/// This will be an iterator of all the values associated with that
/// key.
/// </param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <LongWritable> values, Reducer.Context
context)
{
int theSum = 0;
foreach (LongWritable val in values)
{
theSum += val.Get();
}
sum.Set(theSum);
context.Write(key, sum);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> values,
Reducer.Context context)
{
int seen = 0;
foreach (IntWritable value in values)
{
seen += value.Get();
}
NUnit.Framework.Assert.IsTrue("Bad count for " + key.Get(), Verify(key.Get(), seen
));
context.Write(key, new IntWritable(seen));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString() + this.fieldSeparator;
foreach (Text val in values)
{
FieldSelectionHelper helper = new FieldSelectionHelper();
helper.ExtractOutputKeyValue(keyStr, val.ToString(), fieldSeparator, reduceOutputKeyFieldList
, reduceOutputValueFieldList, allReduceValueFieldsFrom, false, false);
context.Write(helper.GetKey(), helper.GetValue());
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context
context)
{
context.GetCounter("MyCounterGroup", "REDUCE_INPUT_GROUPS").Increment(1);
int sum = 0;
foreach (IntWritable val in values)
{
sum += val.Get();
}
result.Set(sum);
context.Write(key, result);
context.GetCounter("MyCounterGroup", "REDUCE_OUTPUT_RECORDS").Increment(1);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(long key, IEnumerable <string> values, Reducer.Context
context)
{
foreach (string value in values)
{
mos.Write(key, value, value.ToString());
if (!value.ToString().Equals("b"))
{
context.Write(key, value);
}
else
{
mos.Write(Text, key, new Text(Text));
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
foreach (Text value in values)
{
mos.Write(key, value, value.ToString());
if (!value.ToString().Equals("b"))
{
context.Write(key, value);
}
else
{
mos.Write(Text, key, new Text(Text));
mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B"));
mos.Write(Sequence, new IntWritable(3), new Text(Sequence), (Sequence + "_C"));
}
}
}
/// <param name="key">
/// the key is expected to be a Text object, whose prefix indicates
/// the type of aggregation to aggregate the values. In effect, data
/// driven computing is achieved. It is assumed that each aggregator's
/// getReport method emits appropriate output for the aggregator. This
/// may be further customized.
/// </param>
/// <param name="values">the values to be aggregated</param>
/// <param name="context"></param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
string keyStr = key.ToString();
int pos = keyStr.IndexOf(ValueAggregatorDescriptor.TypeSeparator);
string type = Sharpen.Runtime.Substring(keyStr, 0, pos);
keyStr = Sharpen.Runtime.Substring(keyStr, pos + ValueAggregatorDescriptor.TypeSeparator
.Length);
long uniqCount = context.GetConfiguration().GetLong(UniqValueCount.MaxNumUniqueValues
, long.MaxValue);
ValueAggregator aggregator = ValueAggregatorBaseDescriptor.GenerateValueAggregator
(type, uniqCount);
foreach (Text value in values)
{
aggregator.AddNextValue(value);
}
string val = aggregator.GetReport();
key = new Text(keyStr);
context.Write(key, new Text(val));
}