public bool AddJobResultToCache(JobResult jobResult)
{
if (jobResult.jobSucceed == true)
{
string internalCacheKey = "JobResult:"+jobResult.job.GetCacheKey();
return Redis.Instance.DB.StringSet(internalCacheKey, jobResult.resultString);
}
return false;
}
public bool OnJobCalcResult(JobResult jobResult)
{
//No retry for failed job. Retry should be occur only when node is unreachable
JobStatus jobStatus = GetJobStatusByJobID(jobResult.job.jobID);
//mark job status
if (jobStatus != null)
{
if (jobStatus.jobStatusType != JobStatusType.CANCELED)
jobStatus.jobStatusType = JobStatusType.FINISHED;
}
else
{
Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]") + "[Error][OnJobCalcResult] No job status found for jobID=" + jobResult.job.jobID);
return false;
}
//notify node for job finish
//When node is not alive, dont need to pass result to client
Node node = nodeManager.GetNodeByNodeKey(jobResult.job.nodeKey);
if (node != null)
{
node.WorkerFinish(jobResult.job.jobID);
}
else
{
Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]") + "[Error][OnJobCalcResult] No node found for nodeKey=" + jobResult.job.nodeKey + ". Maybe logged out because of hearbeat.");
return false;
}
ProcessWaitingJobQueue();
bool ret = true;
Client client = clientManager.GetClientByClientKey(jobResult.job.clientKey);
if (client != null)
{
AsyncServerClientJobCalcResultCastMessage msg = new AsyncServerClientJobCalcResultCastMessage();
msg.jobResult = jobResult;
if (!client.GetAsyncComm().SendCast(msg))
Console.WriteLine("[Error] SendCast failed!");
ret = true;
}
else
{
ret = false;
}
if (jobResult.resultString.Trim() == "")
{
string str = string.Format("JobCode\n{0}\n--------------------------\nParameter\n{1}\n--------------------------\n{2}\n--------------------------\n", jobResult.job.clientSideJobCode, jobResult.job.parameter, jobResult.resultString);
//File.WriteAllText("empty_ressult.txt", str);
File.AppendAllText("empty_ressult.txt", str);
}
//additionally add to cache
jobResultCache.AddJobResultToCache(jobResult);
return ret;
}
private bool ProcessJobWithCache(Job job)
{
string cacheKey = job.GetCacheKey();
string resultString = jobResultCache.GetJobResultStringFromCache(cacheKey);
if (resultString != null)
{
//cache hit
JobResult jobResult = new JobResult();
jobResult.job = job;
jobResult.jobSucceed = true;
jobResult.resultString = resultString;
JobStatus jobStatus = GetJobStatusByJobID(job.jobID);
//mark job status
if (jobStatus != null)
{
if (jobStatus.jobStatusType != JobStatusType.CANCELED)
jobStatus.jobStatusType = JobStatusType.FINISHED;
}
else
{
Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]") + "[Error][OnJobCalcResult] No job status found for jobID=" + job.jobID);
}
Client client = clientManager.GetClientByClientKey(jobResult.job.clientKey);
if (client != null)
{
AsyncServerClientJobCalcResultCastMessage castmsg = new AsyncServerClientJobCalcResultCastMessage();
castmsg.jobResult = jobResult;
Console.WriteLine("[ProcessJobWithCache] Send job result : " + jobResult.ToString());
if (!client.GetAsyncComm().SendCast(castmsg))
Console.WriteLine("[Error] SendCast failed!");
}
return true;
}
return false;
}
public override void RunJobProcessor(Job job)
{
int maxRunCount = 2;
NativeMethods.SetErrorMode(NativeMethods.SetErrorMode(0) |
ErrorModes.SEM_NOGPFAULTERRORBOX |
ErrorModes.SEM_FAILCRITICALERRORS |
ErrorModes.SEM_NOOPENFILEERRORBOX);
for (int runCount = 1; runCount <= maxRunCount && !jobProcessorManager.IsCanceledJob(job.jobID); runCount++)
{
if (runCount > 1)
{
Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]") + "[Error][JobProcessor] Task start failed. Retry next iteration " + runCount);
Thread.Sleep(1000);
}
//write file
string inputFileName = jobProcessorManager.GetNextFilename() + "_" + job.jobID + ".trn";
string outputFileName = inputFileName + ".res";
string processorFullPath = Path.Combine(Environment.CurrentDirectory, processorRelativePath);
string processorWorkingDirectory = Path.GetDirectoryName(processorFullPath);// Path.Combine(Path.GetDirectoryName(processorFullPath), "tmp");
string IODirectory = Path.Combine(Path.GetDirectoryName(processorFullPath), "tmp");
if (!Directory.Exists(IODirectory))
{
DirectoryInfo di = Directory.CreateDirectory(IODirectory);
}
if (!Directory.Exists(processorWorkingDirectory))
{
DirectoryInfo di = Directory.CreateDirectory(processorWorkingDirectory);
}
string inputFileFullPath = Path.Combine(IODirectory, inputFileName);
string outputFileFullPath = Path.Combine(IODirectory, outputFileName);
Process proc = null;
try
{
File.WriteAllText(inputFileFullPath, job.parameter, Encoding.ASCII);
File.Delete(outputFileFullPath);
string resultStr = null;
proc = jobProcessorManager.GetNewProcess(job);
if (proc == null)
{
if (runCount == maxRunCount)
{
OnJobFailed(job);
return;
}
else
{
continue;
}
}
proc.StartInfo.FileName = processorFullPath;
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.Arguments = "\"" + inputFileFullPath + "\"";
proc.StartInfo.WorkingDirectory = processorWorkingDirectory;
proc.StartInfo.ErrorDialog = false;
if (!proc.Start())
{
if (runCount == maxRunCount)
{
OnJobFailed(job);
return;
}
else
{
continue;
}
}
try
{
proc.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
bool succeed = false;
string line = "";
//starting status
{
JobStatus jobStatus = jobProcessorManager.GetNewJobStatus(job);
jobStatus.jobProgress = 0;
OnJobProgress(jobStatus, null);
}
StringBuilder buffer = new StringBuilder();
while (true)
{
bool lineEndFlag = false;
string currentBuffer;
int outChar = -1;
try
{
outChar = proc.StandardOutput.Read();
if (outChar == -1)
{
//stream closed
if (buffer.Length == 0)
//if stream is empty, do nothing
break;
else
{
//if stream is not empty, process it!
line = buffer.ToString();
lineEndFlag = true;
}
}
else
{
buffer.Append((char)outChar);
currentBuffer = buffer.ToString();
if (currentBuffer.EndsWith("\r\n"))
{
line = currentBuffer.Substring(0, currentBuffer.Length - 2);
lineEndFlag = true;
}
else if (currentBuffer.EndsWith("\n"))
{
line = currentBuffer.Substring(0, currentBuffer.Length - 1);
lineEndFlag = true;
}
}
}
catch (IOException)
{
if (buffer.Length == 0)
//if stream is empty, do nothing
break;
else
{
//if stream is not empty, process it!
line = buffer.ToString();
lineEndFlag = true;
}
}
if (lineEndFlag == true)
{
line.Trim();
buffer.Clear();
//pattern : Progress/currentStep/totalStep (Progress/1/10)
if (line.StartsWith("Progress/"))
{
string[] strArr = line.Split('/');
if (strArr.Length == 3)
{
int curr = int.Parse(strArr[1]);
int total = int.Parse(strArr[2]);
JobStatus jobStatus = jobProcessorManager.GetNewJobStatus(job);
jobStatus.jobProgress = curr / total;
if (OnJobProgress != null)
OnJobProgress(jobStatus, null);
}
}
}
if (outChar == -1)
break;
}
//while ((line = proc.StandardOutput.ReadLine()) != null)
//{
// //Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]")+"[Job=" + job.jobID + "] " + line);
// //pattern : Progress/currentStep/totalStep (Progress/1/10)
// if (line.StartsWith("Progress/"))
// {
// string[] strArr = line.Split('/');
// if (strArr.Length == 3)
// {
// int curr = int.Parse(strArr[1]);
// int total = int.Parse(strArr[2]);
// JobStatus jobStatus = jobProcessorManager.GetNewJobStatus(job);
// jobStatus.jobProgress = curr / total;
// OnJobProgress(jobStatus, null);
// }
// }
//}
proc.WaitForExit();
try
{
resultStr = File.ReadAllText(outputFileFullPath, Encoding.ASCII);
}
catch (Exception)
{
if (runCount == maxRunCount)
{
OnJobFailed(job);
return;
}
else
{
continue;
}
}
JobResult jobResult = new JobResult();
jobResult.job = job;
if (resultStr != null)
{
if (OnJobProgress != null)
{
JobStatus jobStatus = jobProcessorManager.GetNewJobStatus(job);
jobStatus.jobProgress = 1;
OnJobProgress(jobStatus, null);
}
jobResult.jobSucceed = succeed || proc.ExitCode == 0;
jobResult.resultString = "OK" + resultStr;
if (OnJobCompleted != null)
{
OnJobCompleted(jobResult, null);
}
return;
}
else
{
if (runCount == maxRunCount)
{
jobResult.jobSucceed = proc.ExitCode == 0;
OnJobCompleted(jobResult, null);
return;
}
else
{
continue;
}
}
}
catch (Exception e)
{
if (proc != null)
proc.Close();
Console.WriteLine(DateTime.Now.ToString("[HH:mm:ss.fff]") + "[Error][JobProcessor] Task start failed." + e.ToString());
if (runCount == maxRunCount)
{
OnJobFailed(job);
return;
}
continue;
}
}
if (jobProcessorManager.IsCanceledJob(job.jobID))
OnJobFailed(job);
return;
}
private void OnJobFailed(Job job)
{
JobResult jobResult = new JobResult();
jobResult.job = job;
jobResult.jobSucceed = false;
jobResult.resultString = "OKfail";
if (OnJobCompleted != null)
OnJobCompleted(jobResult, null);
}
