/// <summary>Sets up a job conf for the given job using the given config object.</summary>
/// <remarks>
/// Sets up a job conf for the given job using the given config object. Ensures
/// that the correct input format is set, the mapper and and reducer class and
/// the input and output keys and value classes along with any other job
/// configuration.
/// </remarks>
/// <param name="config"/>
/// <returns>JobConf representing the job to be ran</returns>
/// <exception cref="System.IO.IOException"/>
private JobConf GetJob(ConfigExtractor config)
{
JobConf job = new JobConf(config.GetConfig(), typeof(Org.Apache.Hadoop.FS.Slive.SliveTest
));
job.SetInputFormat(typeof(DummyInputFormat));
FileOutputFormat.SetOutputPath(job, config.GetOutputPath());
job.SetMapperClass(typeof(SliveMapper));
job.SetPartitionerClass(typeof(SlivePartitioner));
job.SetReducerClass(typeof(SliveReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetOutputFormat(typeof(TextOutputFormat));
TextOutputFormat.SetCompressOutput(job, false);
job.SetNumReduceTasks(config.GetReducerAmount());
job.SetNumMapTasks(config.GetMapAmount());
return(job);
}
/// <summary>
/// Attempts to write the report to the given output using the specified
/// config.
/// </summary>
/// <remarks>
/// Attempts to write the report to the given output using the specified
/// config. It will open up the expected reducer output file and read in its
/// contents and then split up by operation output and sort by operation type
/// and then for each operation type it will generate a report to the specified
/// result file and the console.
/// </remarks>
/// <param name="cfg">the config specifying the files and output</param>
/// <exception cref="System.Exception">if files can not be opened/closed/read or invalid format
/// </exception>
private void WriteReport(ConfigExtractor cfg)
{
Path dn = cfg.GetOutputPath();
Log.Info("Writing report using contents of " + dn);
FileSystem fs = dn.GetFileSystem(cfg.GetConfig());
FileStatus[] reduceFiles = fs.ListStatus(dn);
BufferedReader fileReader = null;
PrintWriter reportWriter = null;
try
{
IList <OperationOutput> noOperations = new AList <OperationOutput>();
IDictionary <string, IList <OperationOutput> > splitTypes = new SortedDictionary <string
, IList <OperationOutput> >();
foreach (FileStatus fn in reduceFiles)
{
if (!fn.GetPath().GetName().StartsWith("part"))
{
continue;
}
fileReader = new BufferedReader(new InputStreamReader(new DataInputStream(fs.Open
(fn.GetPath()))));
string line;
while ((line = fileReader.ReadLine()) != null)
{
string[] pieces = line.Split("\t", 2);
if (pieces.Length == 2)
{
OperationOutput data = new OperationOutput(pieces[0], pieces[1]);
string op = (data.GetOperationType());
if (op != null)
{
IList <OperationOutput> opList = splitTypes[op];
if (opList == null)
{
opList = new AList <OperationOutput>();
}
opList.AddItem(data);
splitTypes[op] = opList;
}
else
{
noOperations.AddItem(data);
}
}
else
{
throw new IOException("Unparseable line " + line);
}
}
fileReader.Close();
fileReader = null;
}
FilePath resFile = null;
if (cfg.GetResultFile() != null)
{
resFile = new FilePath(cfg.GetResultFile());
}
if (resFile != null)
{
Log.Info("Report results being placed to logging output and to file " + resFile.GetCanonicalPath
());
reportWriter = new PrintWriter(new FileOutputStream(resFile));
}
else
{
Log.Info("Report results being placed to logging output");
}
ReportWriter reporter = new ReportWriter();
if (!noOperations.IsEmpty())
{
reporter.BasicReport(noOperations, reportWriter);
}
foreach (string opType in splitTypes.Keys)
{
reporter.OpReport(opType, splitTypes[opType], reportWriter);
}
}
finally
{
if (fileReader != null)
{
fileReader.Close();
}
if (reportWriter != null)
{
reportWriter.Close();
}
}
}
