/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
bool oldConfigs = conf.GetBoolean(OldConfigs, false);
if (oldConfigs)
{
string javaOpts = conf.Get(JobConf.MapredTaskJavaOpts);
NUnit.Framework.Assert.IsNotNull(JobConf.MapredTaskJavaOpts + " is null!", javaOpts
);
NUnit.Framework.Assert.AreEqual(JobConf.MapredTaskJavaOpts + " has value of: " +
javaOpts, javaOpts, TaskOptsVal);
}
else
{
string reduceJavaOpts = conf.Get(JobConf.MapredReduceTaskJavaOpts);
NUnit.Framework.Assert.IsNotNull(JobConf.MapredReduceTaskJavaOpts + " is null!",
reduceJavaOpts);
NUnit.Framework.Assert.AreEqual(JobConf.MapredReduceTaskJavaOpts + " has value of: "
+ reduceJavaOpts, reduceJavaOpts, ReduceOptsVal);
}
Level logLevel = Level.ToLevel(conf.Get(JobConf.MapredReduceTaskLogLevel, Level.Info
.ToString()));
NUnit.Framework.Assert.AreEqual(JobConf.MapredReduceTaskLogLevel + "has value of "
+ logLevel, logLevel, Level.Off);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Reducer.Context context)
{
Configuration conf = context.GetConfiguration();
this.reduceSleepCount = conf.GetInt(ReduceSleepCount, reduceSleepCount);
this.reduceSleepDuration = reduceSleepCount == 0 ? 0 : conf.GetLong(ReduceSleepTime
, 100) / reduceSleepCount;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(K key, IEnumerable <V> values, Reducer.Context context
)
{
foreach (V value in values)
{
Emit(key, value, context);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void Reduce(Text key, IEnumerator <Text> values, Reducer.Context context
)
{
float reducePhaseProgress = ((float)++recordCount) / InputLines;
float weightedReducePhaseProgress = ReduceProgressRange * reducePhaseProgress;
NUnit.Framework.Assert.AreEqual("Invalid progress in reduce", ShuffleProgressRange
+ weightedReducePhaseProgress, context.GetProgress(), 0.01f);
}
/// <exception cref="System.IO.IOException"/>
protected override void Reduce(LongWritable key, IEnumerable <NullWritable> values
, Reducer.Context context)
{
if (context.GetConfiguration().GetBoolean(FailReduce, false))
{
throw new RuntimeException("Intentional reduce failure");
}
context.SetStatus("No worries");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void Emit(K key, V val, Reducer.Context context)
{
++total;
while ((float)kept / total < keep)
{
++kept;
context.Write(key, val);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Reducer.Context context)
{
// Note that the reduce will read some segments before calling setup()
float reducePhaseProgress = ((float)++recordCount) / InputLines;
float weightedReducePhaseProgress = ReduceProgressRange * reducePhaseProgress;
// check that the shuffle phase progress is accounted for
NUnit.Framework.Assert.AreEqual("Invalid progress in reduce setup", ShuffleProgressRange
+ weightedReducePhaseProgress, context.GetProgress(), 0.01f);
}
/// <exception cref="System.IO.IOException"/>
protected override void Reduce(IntWritable key, IEnumerable <NullWritable> values,
Reducer.Context context)
{
try
{
context.SetStatus("Sleeping... (" + (reduceSleepDuration * (reduceSleepCount - count
)) + ") ms left");
Sharpen.Thread.Sleep(reduceSleepDuration);
}
catch (Exception ex)
{
throw (IOException)Sharpen.Extensions.InitCause(new IOException("Interrupted while sleeping"
), ex);
}
count++;
}
// NOTHING
/// <summary>
/// Advanced application writers can use the
/// <see cref="Reducer{KEYIN, VALUEIN, KEYOUT, VALUEOUT}.Run(Context)"/>
/// method to
/// control how the reduce task works.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void Run(Reducer.Context context)
{
Setup(context);
try
{
while (context.NextKey())
{
Reduce(context.GetCurrentKey(), context.GetValues(), context);
// If a back up store is used, reset it
IEnumerator <VALUEIN> iter = context.GetValues().GetEnumerator();
if (iter is ReduceContext.ValueIterator)
{
((ReduceContext.ValueIterator <VALUEIN>)iter).ResetBackupStore();
}
}
}
finally
{
Cleanup(context);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> values,
Reducer.Context context)
{
int errors = 0;
MarkableIterator <IntWritable> mitr = new MarkableIterator <IntWritable>(values.GetEnumerator
());
switch (key.Get())
{
case 0:
{
errors += Test0(key, mitr);
break;
}
case 1:
{
errors += Test1(key, mitr);
break;
}
case 2:
{
errors += Test2(key, mitr);
break;
}
case 3:
{
errors += Test3(key, mitr);
break;
}
default:
{
break;
}
}
context.Write(key, new IntWritable(errors));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> vals, Reducer.Context
context)
{
long @out = 0;
foreach (Text val in vals)
{
@out++;
}
context.Write(key, new LongWritable(@out));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> values, Reducer.Context
context)
{
string id = context.GetTaskAttemptID().ToString();
// Reducer 0 does not output anything
if (!id.EndsWith("0_0"))
{
foreach (Text val in values)
{
context.Write(key, val);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <LongWritable> values, Reducer.Context
context)
{
LongWritable maxValue = null;
foreach (LongWritable value in values)
{
if (maxValue == null)
{
maxValue = value;
}
else
{
if (value.CompareTo(maxValue) > 0)
{
maxValue = value;
}
}
}
context.Write(key, maxValue);
}
/// <summary>Called once at the end of the task.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void Cleanup(Reducer.Context context)
{
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> it, Reducer.Context
context)
{
foreach (IntWritable iw in it)
{
context.Write(iw, null);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void Reduce(IntWritable key, IEnumerator <IntWritable> it, Reducer.Context
context)
{
int keyint = key.Get();
int total = 0;
while (it.HasNext())
{
total += it.Next().Get();
}
context.Write(new IntWritable(keyint), new IntWritable(total));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void Reduce(Org.Apache.Hadoop.IO.Text key, IEnumerator <Org.Apache.Hadoop.IO.Text
> values, Reducer.Context context)
{
Org.Apache.Hadoop.IO.Text dumbKey = new Org.Apache.Hadoop.IO.Text(string.Empty);
while (values.HasNext())
{
Org.Apache.Hadoop.IO.Text data = values.Next();
context.Write(dumbKey, data);
}
}
/// <summary>Called once at the start of the task.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void Setup(Reducer.Context context)
{
}
protected override void Setup(Reducer.Context context)
{
this.keep = context.GetConfiguration().GetFloat(ReducePreservePercent, (float)100.0
) / (float)100.0;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
NUnit.Framework.Assert.AreEqual("Unexpected record count", expected, numrecs);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(LongWritable key, IEnumerable <Text> vals, Reducer.Context
context)
{
throw new IOException("TestMapperReducerCleanup");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(TestMapCollection.KeyWritable k, IEnumerable <TestMapCollection.ValWritable
> values, Reducer.Context context)
{
foreach (TestMapCollection.ValWritable val in values)
{
++numrecs;
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(IntWritable key, IEnumerable <IntWritable> it, Reducer.Context
context)
{
int keyint = key.Get();
int count = 0;
foreach (IntWritable iw in it)
{
count++;
}
context.Write(new IntWritable(keyint), new IntWritable(count));
}
// NOTHING
/// <summary>This method is called once for each key.</summary>
/// <remarks>
/// This method is called once for each key. Most applications will define
/// their reduce class by overriding this method. The default implementation
/// is an identity function.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void Reduce(KEYIN key, IEnumerable <VALUEIN> values, Reducer.Context
context)
{
foreach (VALUEIN value in values)
{
context.Write((KEYOUT)key, (VALUEOUT)value);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(BytesWritable key, IEnumerable <BytesWritable> values
, Reducer.Context context)
{
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
// check if the reduce task has progress of 1 in the end
NUnit.Framework.Assert.AreEqual("Invalid progress in reduce cleanup", 1.0f, context
.GetProgress(), 0f);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Reducer.Context context)
{
reduceCleanup = true;
base.Cleanup(context);
}
protected override void Setup(Reducer.Context job)
{
numrecs = 0;
expected = job.GetConfiguration().GetInt("test.spillmap.records", 100);
}