/*
* (non-Javadoc)
*
* @see org.apache.hadoop.mapred.Reducer#reduce(java.lang.Object,
* java.util.Iterator, org.apache.hadoop.mapred.OutputCollector,
* org.apache.hadoop.mapred.Reporter)
*/
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(Text key, IEnumerator <Text> values, OutputCollector <Text
, Text> output, Reporter reporter)
{
// Reducer
OperationOutput collector = null;
int reduceAm = 0;
int errorAm = 0;
LogAndSetStatus(reporter, "Iterating over reduction values for key " + key);
while (values.HasNext())
{
Text value = values.Next();
try
{
OperationOutput val = new OperationOutput(key, value);
if (collector == null)
{
collector = val;
}
else
{
collector = OperationOutput.Merge(collector, val);
}
Log.Info("Combined " + val + " into/with " + collector);
++reduceAm;
}
catch (Exception e)
{
++errorAm;
LogAndSetStatus(reporter, "Error iterating over reduction input " + value + " due to : "
+ StringUtils.StringifyException(e));
if (GetConfig().ShouldExitOnFirstError())
{
break;
}
}
}
LogAndSetStatus(reporter, "Reduced " + reduceAm + " values with " + errorAm + " errors"
);
if (collector != null)
{
LogAndSetStatus(reporter, "Writing output " + collector.GetKey() + " : " + collector
.GetOutputValue());
output.Collect(collector.GetKey(), collector.GetOutputValue());
}
}
/// <summary>Provides a more detailed report for a given operation.</summary>
/// <remarks>
/// Provides a more detailed report for a given operation. This will output the
/// keys and values for all input and then sort based on measurement type and
/// attempt to show rates for various metrics which have expected types to be
/// able to measure there rate. Currently this will show rates for bytes
/// written, success count, files created, directory entries, op count and
/// bytes read if the variable for time taken is available for each measurement
/// type.
/// </remarks>
/// <param name="operation">the operation that is being reported on.</param>
/// <param name="input">the set of data for that that operation.</param>
/// <param name="os">
/// any print writer for which output should be written to (along with
/// the logging library)
/// </param>
internal virtual void OpReport(string operation, IList <OperationOutput> input, PrintWriter
os)
{
WriteMessage("Basic report for operation type " + operation, os);
WriteMessage(GetSectionDelimiter(), os);
foreach (OperationOutput data in input)
{
WriteMessage("Measurement \"" + data.GetMeasurementType() + "\" = " + data.GetValue
(), os);
}
// split up into measurement types for rates...
IDictionary <string, OperationOutput> combined = new SortedDictionary <string, OperationOutput
>();
foreach (OperationOutput data_1 in input)
{
if (combined.Contains(data_1.GetMeasurementType()))
{
OperationOutput curr = combined[data_1.GetMeasurementType()];
combined[data_1.GetMeasurementType()] = OperationOutput.Merge(curr, data_1);
}
else
{
combined[data_1.GetMeasurementType()] = data_1;
}
}
// handle the known types
OperationOutput timeTaken = combined[OkTimeTaken];
if (timeTaken != null)
{
long mTaken = long.Parse(timeTaken.GetValue().ToString());
if (mTaken > 0)
{
NumberFormat formatter = Formatter.GetDecimalFormatter();
foreach (string measurementType in combined.Keys)
{
double rate = null;
string rateType = string.Empty;
if (measurementType.Equals(BytesWritten))
{
long mbWritten = long.Parse(combined[measurementType].GetValue().ToString()) / (Constants
.Megabytes);
rate = (double)mbWritten / (double)(mTaken / 1000.0d);
rateType = "MB/sec";
}
else
{
if (measurementType.Equals(Successes))
{
long succ = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)succ / (double)(mTaken / 1000.0d);
rateType = "successes/sec";
}
else
{
if (measurementType.Equals(FilesCreated))
{
long filesCreated = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)filesCreated / (double)(mTaken / 1000.0d);
rateType = "files created/sec";
}
else
{
if (measurementType.Equals(DirEntries))
{
long entries = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)entries / (double)(mTaken / 1000.0d);
rateType = "directory entries/sec";
}
else
{
if (measurementType.Equals(OpCount))
{
long opCount = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)opCount / (double)(mTaken / 1000.0d);
rateType = "operations/sec";
}
else
{
if (measurementType.Equals(BytesRead))
{
long mbRead = long.Parse(combined[measurementType].GetValue().ToString()) / (Constants
.Megabytes);
rate = (double)mbRead / (double)(mTaken / 1000.0d);
rateType = "MB/sec";
}
}
}
}
}
}
if (rate != null)
{
WriteMessage("Rate for measurement \"" + measurementType + "\" = " + formatter.Format
(rate) + " " + rateType, os);
}
}
}
}
WriteMessage(GetSectionDelimiter(), os);
}