/// <exception cref="System.Exception"/>
private string ReadStdOut(JobConf conf)
{
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig
.TaskAttemptId)));
FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout);
return(ReadFile(stdOut));
}
/// <summary>clean previous std error and outs</summary>
private void InitStdOut(JobConf configuration)
{
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(configuration.Get(MRJobConfig
.TaskAttemptId)));
FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout);
FilePath stdErr = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stderr);
// prepare folder
if (!stdOut.GetParentFile().Exists())
{
stdOut.GetParentFile().Mkdirs();
}
else
{
// clean logs
stdOut.DeleteOnExit();
stdErr.DeleteOnExit();
}
}
/// <summary>Start the child process to handle the task for us.</summary>
/// <param name="conf">the task's configuration</param>
/// <param name="recordReader">the fake record reader to update progress with</param>
/// <param name="output">the collector to send output to</param>
/// <param name="reporter">the reporter for the task</param>
/// <param name="outputKeyClass">the class of the output keys</param>
/// <param name="outputValueClass">the class of the output values</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal Application(JobConf conf, RecordReader <FloatWritable, NullWritable> recordReader
, OutputCollector <K2, V2> output, Reporter reporter, Type outputKeyClass, Type outputValueClass
)
{
serverSocket = Sharpen.Extensions.CreateServerSocket(0);
IDictionary <string, string> env = new Dictionary <string, string>();
// add TMPDIR environment variable with the value of java.io.tmpdir
env["TMPDIR"] = Runtime.GetProperty("java.io.tmpdir");
env[Submitter.Port] = Sharpen.Extensions.ToString(serverSocket.GetLocalPort());
//Add token to the environment if security is enabled
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = TokenCache.
GetJobToken(conf.GetCredentials());
// This password is used as shared secret key between this application and
// child pipes process
byte[] password = jobToken.GetPassword();
string localPasswordFile = new FilePath(".") + Path.Separator + "jobTokenPassword";
WritePasswordToLocalFile(localPasswordFile, password, conf);
env["hadoop.pipes.shared.secret.location"] = localPasswordFile;
IList <string> cmd = new AList <string>();
string interpretor = conf.Get(Submitter.Interpretor);
if (interpretor != null)
{
cmd.AddItem(interpretor);
}
string executable = DistributedCache.GetLocalCacheFiles(conf)[0].ToString();
if (!FileUtil.CanExecute(new FilePath(executable)))
{
// LinuxTaskController sets +x permissions on all distcache files already.
// In case of DefaultTaskController, set permissions here.
FileUtil.Chmod(executable, "u+x");
}
cmd.AddItem(executable);
// wrap the command in a stdout/stderr capture
// we are starting map/reduce task of the pipes job. this is not a cleanup
// attempt.
TaskAttemptID taskid = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig
.TaskAttemptId)));
FilePath stdout = TaskLog.GetTaskLogFile(taskid, false, TaskLog.LogName.Stdout);
FilePath stderr = TaskLog.GetTaskLogFile(taskid, false, TaskLog.LogName.Stderr);
long logLength = TaskLog.GetTaskLogLength(conf);
cmd = TaskLog.CaptureOutAndError(null, cmd, stdout, stderr, logLength, false);
process = RunClient(cmd, env);
clientSocket = serverSocket.Accept();
string challenge = GetSecurityChallenge();
string digestToSend = CreateDigest(password, challenge);
string digestExpected = CreateDigest(password, digestToSend);
handler = new OutputHandler <K2, V2>(output, reporter, recordReader, digestExpected
);
K2 outputKey = (K2)ReflectionUtils.NewInstance(outputKeyClass, conf);
V2 outputValue = (V2)ReflectionUtils.NewInstance(outputValueClass, conf);
downlink = new BinaryProtocol <K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue
, conf);
downlink.Authenticate(digestToSend, challenge);
WaitForAuthentication();
Log.Debug("Authentication succeeded");
downlink.Start();
downlink.SetJobConf(conf);
}