private void CompleteWithStatus(byte status)
{
lock (jobsDataContext)
{
try
{
//Trace.WriteLine(string.Format("setting job {0} status {1}",this,status));
jobsDataContext.Refresh(RefreshMode.KeepChanges, job);
job.Status = status;
jobsDataContext.SubmitChanges();
}
catch (ChangeConflictException)
{
foreach (var item in jobsDataContext.ChangeConflicts)
{
if (item.IsDeleted)
{
JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":The job was unexpectedly deleted");
}
foreach (var conflict in item.MemberConflicts)
{
JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":{4} Conflict {3} original:{0} current:{1} database:{2}", conflict.OriginalValue, conflict.CurrentValue, conflict.DatabaseValue, conflict.Member.Name, this);
}
}
throw;
}
finally
{
stopTouch.Set();
}
}
}
private void TouchThreadFunc(object state)
{
Job job = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item1;
JobsDBDataContext jobsDataContext = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item2;
ManualResetEvent stopEvent = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item3;
var ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes");
long curWorkingSet, curGCTotal, curPrivate;
long maxWorkingSet = 0;
long maxGCTotal = 0;
long maxPrivate = 0;
double curRamAvailMb, minRamAvailMb = double.MaxValue;
int fails = 0;
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread running");
bool jobDeleted = false;
while (true)
{
Process currentProcess = Process.GetCurrentProcess();
currentProcess.Refresh();
curWorkingSet = Environment.WorkingSet;
curGCTotal = GC.GetTotalMemory(false);
curPrivate = currentProcess.PrivateMemorySize64;
curRamAvailMb = ramCounter.NextValue();
if (maxWorkingSet < curWorkingSet)
{
maxWorkingSet = curWorkingSet;
}
if (maxGCTotal < curGCTotal)
{
maxGCTotal = curGCTotal;
}
if (maxPrivate < curPrivate)
{
maxPrivate = curPrivate;
}
if (minRamAvailMb > curRamAvailMb)
{
minRamAvailMb = curRamAvailMb;
}
try
{
bool cancelationCheckNeeded = false;
lock (jobsDataContext)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Touch thread tick", job);
bool touched = false;
for (int touchTries = 0; touchTries < 5; touchTries++)
{
try
{
jobsDataContext.Refresh(RefreshMode.KeepChanges, job);
job.Touchtime = DateTime.UtcNow;
jobsDataContext.SubmitChanges();
touched = true;
break;
}
catch (ChangeConflictException)
{
//TraceConflicts(jobsDataContext, "updating touch time");
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to update touchtime. attempt {1}", job, touchTries + 1);
jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues);
}
}
if (touched)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Updated TouchTime", job);
}
else
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Failed to update TouchTime", job);
}
if (job.IsHeavyJob == 0)
{
var execTimespanSec = (DateTime.UtcNow - job.StartTime.Value).TotalSeconds;
if (execTimespanSec > settings.LightJobExecutionPermitedTimeSec)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Job considered to be HEAVY", job);
for (int retries = 0; retries < 10; retries++)
{
try
{
var allParts = jobsDataContext.Jobs.Where(j1 => j1.Hash == job.Hash).ToArray();
foreach (var j in allParts)
{
j.IsHeavyJob = 1;
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Committing HEAVY flag for all jobs part", job);
jobsDataContext.Refresh(RefreshMode.KeepChanges, allParts);
jobsDataContext.SubmitChanges(ConflictMode.ContinueOnConflict);
break;
}
catch (ChangeConflictException)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to set heavy flag for all parts. attempt {1}", job, retries + 1);
//TraceConflicts(jobsDataContext, "heavy flag committing");
jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues);
}
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Successfully committed HEAVY flag for all jobs part", job);
cancelationCheckNeeded = true;
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job still light ({1} execution seconds of {2} permitted)", job, execTimespanSec, settings.LightJobExecutionPermitedTimeSec);
}
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job is already marked as heavy", job);
cancelationCheckNeeded = true;
}
if (cancelationCheckNeeded)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Checking for heavy job cancelation", job);
int otherRunningHeavyPartsCount = jobsDataContext.Jobs.Count(j2 => j2.StartTime < job.StartTime && j2.Status == 1 && j2.IsHeavyJob == 1);
if (otherRunningHeavyPartsCount >= settings.PermitedHeavyPartWorkers)
{
JobManager.JobManagerTrace.TraceInfo(this.ToShortString() + ":{0} Aborting working thread as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers);
if (workingThread != null)
{
workingThread.Abort();
}
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Continuing heavy part as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers);
}
}
}
if (stopEvent.WaitOne(TimeSpan.FromSeconds(settings.TouchPeriodInSeconds)))
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} detected touch thread stop request. stopping touch thread loop...", job);
break;
}
}
catch (InvalidOperationException exc)
{
jobDeleted = true;
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} InvalidOperationException. Is job part deleted? ({1})", job, exc.ToString());
break;
}
catch (ChangeConflictException)
{
lock (jobsDataContext)
{
if (TraceConflicts(jobsDataContext, "last chance exc") || jobDeleted)
{
break;
}
}
}
catch (Exception exc)
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Error updating touch time for job {1}", job, exc.ToString());
if (fails++ > 3)
{
break;
}
}
if (jobDeleted)
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} record is deleted. stoping touch thread", job);
}
//curPhys = currentProcess.WorkingSet64;
//curPaged = currentProcess.PagedMemorySize64;
//curVirtual = currentProcess.VirtualMemorySize64;
//if (currentLogRecord.PeakPhysicalMemoryUsage < curPhys) currentLogRecord.PeakPhysicalMemoryUsage = curPhys;
//if (currentLogRecord.PeakPagedMemoryUsage < curPaged) currentLogRecord.PeakPagedMemoryUsage = curPaged;
//if (currentLogRecord.PeakVirtualMemoryUsage < curVirtual) currentLogRecord.PeakVirtualMemoryUsage = curVirtual;
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread ending. Peak mem stats: working set {0}Mb, GC.allocated {1}Mb, PrivateMem {2}Mb, min available Ram {3}Mb",
((double)maxWorkingSet) / (1024 * 1024), ((double)maxGCTotal) / (1024 * 1024), ((double)maxPrivate) / (1024 * 1024), minRamAvailMb);
//currentLogRecord.WorkEnd = DateTime.UtcNow;
//logManager.Insert(currentLogRecord);
}
public JobStatus Submit(IFetchRequest request, string hash, double jobRegistrationPermitedTime, int minPointPerPartition, int maxPointPerPartition, int totalWorkers)
{
var jobStatus = GetStatus(hash, false);
int retries = 0;
while (true)
{
if (jobStatus == null)
{
JobManagerTrace.TraceVerbose("{0}: Request is new (not in jobs table)", hash);
// Try to create blob
try
{
if (CreateBlobWithJob(request, hash, jobRegistrationPermitedTime)) // Successfully created
{
DateTime currentUTC = DateTime.UtcNow;
int partsCount = GetPartitionsCount(request, minPointPerPartition, maxPointPerPartition, GetFreeWorkersNo(totalWorkers));
var partsToInsert = Enumerable.Range(0, partsCount).Select(partNumber => new Job()
{
Hash = hash, PartNo = partNumber, PartsCount = partsCount, Status = (byte)JobOrPartState.Pending, Priority = 0, SubmitTime = currentUTC, Touchtime = currentUTC
}).ToArray();
try
{
jobsDataContext.Jobs.InsertAllOnSubmit(partsToInsert);
jobsDataContext.SubmitChanges(System.Data.Linq.ConflictMode.FailOnFirstConflict);
}
catch (Exception ex)
{
JobManagerTrace.TraceError("{0}:Exception during insertion of job records into the job table. Reverting job registration (deleting job blob, all job records) {1}", hash, ex.ToString());
DiscardDataContextPendingChanges();
jobsDataContext.ExecuteCommand("delete from Job where hash='{0}'", hash); //order is important. deleting records before blob!
DeleteJobBlob(hash);
throw; //to be caught by retrying code
}
for (int i = 1; i <= partsCount; ++i)
{
JobManagerTrace.TraceVerbose("{0}:{1}:{2}:Record created in the jobs table", hash, i, partsCount);
}
return(GetStatus(hash));
}
else // Blob exists
{
jobStatus = JobStatus.GetPendingStatus(hash, Int32.MaxValue); // Unknown queue length
JobManagerTrace.TraceVerbose("{0}: Blob for request is created recently by someone else. Returning {1} status.", hash, jobStatus);
return(jobStatus);
}
}
catch (Exception e)
{
JobManagerTrace.TraceError("{0}:Request submitting error: {1}", hash, e.ToString());
if (retries++ > 3)
{
JobManagerTrace.TraceError("{0}:3 retries failed for request. Reporting failed state to the client", hash);
return(JobStatus.GetFailedStatus(e.ToString()));
}
}
}
else
{
JobManagerTrace.TraceVerbose("{0}: Request is already submitted. Current state is {1}", hash, jobStatus);
return(jobStatus);
}
}
}