/// <exception cref="System.Exception"/> private string ReadStdOut(JobConf conf) { TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig .TaskAttemptId))); FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout); return(ReadFile(stdOut)); }
/// <summary>clean previous std error and outs</summary> private void InitStdOut(JobConf configuration) { TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(configuration.Get(MRJobConfig .TaskAttemptId))); FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout); FilePath stdErr = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stderr); // prepare folder if (!stdOut.GetParentFile().Exists()) { stdOut.GetParentFile().Mkdirs(); } else { // clean logs stdOut.DeleteOnExit(); stdErr.DeleteOnExit(); } }
/// <summary>Start the child process to handle the task for us.</summary> /// <param name="conf">the task's configuration</param> /// <param name="recordReader">the fake record reader to update progress with</param> /// <param name="output">the collector to send output to</param> /// <param name="reporter">the reporter for the task</param> /// <param name="outputKeyClass">the class of the output keys</param> /// <param name="outputValueClass">the class of the output values</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal Application(JobConf conf, RecordReader <FloatWritable, NullWritable> recordReader , OutputCollector <K2, V2> output, Reporter reporter, Type outputKeyClass, Type outputValueClass ) { serverSocket = Sharpen.Extensions.CreateServerSocket(0); IDictionary <string, string> env = new Dictionary <string, string>(); // add TMPDIR environment variable with the value of java.io.tmpdir env["TMPDIR"] = Runtime.GetProperty("java.io.tmpdir"); env[Submitter.Port] = Sharpen.Extensions.ToString(serverSocket.GetLocalPort()); //Add token to the environment if security is enabled Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = TokenCache. GetJobToken(conf.GetCredentials()); // This password is used as shared secret key between this application and // child pipes process byte[] password = jobToken.GetPassword(); string localPasswordFile = new FilePath(".") + Path.Separator + "jobTokenPassword"; WritePasswordToLocalFile(localPasswordFile, password, conf); env["hadoop.pipes.shared.secret.location"] = localPasswordFile; IList <string> cmd = new AList <string>(); string interpretor = conf.Get(Submitter.Interpretor); if (interpretor != null) { cmd.AddItem(interpretor); } string executable = DistributedCache.GetLocalCacheFiles(conf)[0].ToString(); if (!FileUtil.CanExecute(new FilePath(executable))) { // LinuxTaskController sets +x permissions on all distcache files already. // In case of DefaultTaskController, set permissions here. FileUtil.Chmod(executable, "u+x"); } cmd.AddItem(executable); // wrap the command in a stdout/stderr capture // we are starting map/reduce task of the pipes job. this is not a cleanup // attempt. TaskAttemptID taskid = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig .TaskAttemptId))); FilePath stdout = TaskLog.GetTaskLogFile(taskid, false, TaskLog.LogName.Stdout); FilePath stderr = TaskLog.GetTaskLogFile(taskid, false, TaskLog.LogName.Stderr); long logLength = TaskLog.GetTaskLogLength(conf); cmd = TaskLog.CaptureOutAndError(null, cmd, stdout, stderr, logLength, false); process = RunClient(cmd, env); clientSocket = serverSocket.Accept(); string challenge = GetSecurityChallenge(); string digestToSend = CreateDigest(password, challenge); string digestExpected = CreateDigest(password, digestToSend); handler = new OutputHandler <K2, V2>(output, reporter, recordReader, digestExpected ); K2 outputKey = (K2)ReflectionUtils.NewInstance(outputKeyClass, conf); V2 outputValue = (V2)ReflectionUtils.NewInstance(outputValueClass, conf); downlink = new BinaryProtocol <K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue , conf); downlink.Authenticate(digestToSend, challenge); WaitForAuthentication(); Log.Debug("Authentication succeeded"); downlink.Start(); downlink.SetJobConf(conf); }