private void CompleteWithStatus(byte status) { lock (jobsDataContext) { try { //Trace.WriteLine(string.Format("setting job {0} status {1}",this,status)); jobsDataContext.Refresh(RefreshMode.KeepChanges, job); job.Status = status; jobsDataContext.SubmitChanges(); } catch (ChangeConflictException) { foreach (var item in jobsDataContext.ChangeConflicts) { if (item.IsDeleted) { JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":The job was unexpectedly deleted"); } foreach (var conflict in item.MemberConflicts) { JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":{4} Conflict {3} original:{0} current:{1} database:{2}", conflict.OriginalValue, conflict.CurrentValue, conflict.DatabaseValue, conflict.Member.Name, this); } } throw; } finally { stopTouch.Set(); } } }
private void TouchThreadFunc(object state) { Job job = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item1; JobsDBDataContext jobsDataContext = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item2; ManualResetEvent stopEvent = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item3; var ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes"); long curWorkingSet, curGCTotal, curPrivate; long maxWorkingSet = 0; long maxGCTotal = 0; long maxPrivate = 0; double curRamAvailMb, minRamAvailMb = double.MaxValue; int fails = 0; JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread running"); bool jobDeleted = false; while (true) { Process currentProcess = Process.GetCurrentProcess(); currentProcess.Refresh(); curWorkingSet = Environment.WorkingSet; curGCTotal = GC.GetTotalMemory(false); curPrivate = currentProcess.PrivateMemorySize64; curRamAvailMb = ramCounter.NextValue(); if (maxWorkingSet < curWorkingSet) { maxWorkingSet = curWorkingSet; } if (maxGCTotal < curGCTotal) { maxGCTotal = curGCTotal; } if (maxPrivate < curPrivate) { maxPrivate = curPrivate; } if (minRamAvailMb > curRamAvailMb) { minRamAvailMb = curRamAvailMb; } try { bool cancelationCheckNeeded = false; lock (jobsDataContext) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Touch thread tick", job); bool touched = false; for (int touchTries = 0; touchTries < 5; touchTries++) { try { jobsDataContext.Refresh(RefreshMode.KeepChanges, job); job.Touchtime = DateTime.UtcNow; jobsDataContext.SubmitChanges(); touched = true; break; } catch (ChangeConflictException) { //TraceConflicts(jobsDataContext, "updating touch time"); JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to update touchtime. attempt {1}", job, touchTries + 1); jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues); } } if (touched) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Updated TouchTime", job); } else { JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Failed to update TouchTime", job); } if (job.IsHeavyJob == 0) { var execTimespanSec = (DateTime.UtcNow - job.StartTime.Value).TotalSeconds; if (execTimespanSec > settings.LightJobExecutionPermitedTimeSec) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Job considered to be HEAVY", job); for (int retries = 0; retries < 10; retries++) { try { var allParts = jobsDataContext.Jobs.Where(j1 => j1.Hash == job.Hash).ToArray(); foreach (var j in allParts) { j.IsHeavyJob = 1; } JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Committing HEAVY flag for all jobs part", job); jobsDataContext.Refresh(RefreshMode.KeepChanges, allParts); jobsDataContext.SubmitChanges(ConflictMode.ContinueOnConflict); break; } catch (ChangeConflictException) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to set heavy flag for all parts. attempt {1}", job, retries + 1); //TraceConflicts(jobsDataContext, "heavy flag committing"); jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues); } } JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Successfully committed HEAVY flag for all jobs part", job); cancelationCheckNeeded = true; } else { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job still light ({1} execution seconds of {2} permitted)", job, execTimespanSec, settings.LightJobExecutionPermitedTimeSec); } } else { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job is already marked as heavy", job); cancelationCheckNeeded = true; } if (cancelationCheckNeeded) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Checking for heavy job cancelation", job); int otherRunningHeavyPartsCount = jobsDataContext.Jobs.Count(j2 => j2.StartTime < job.StartTime && j2.Status == 1 && j2.IsHeavyJob == 1); if (otherRunningHeavyPartsCount >= settings.PermitedHeavyPartWorkers) { JobManager.JobManagerTrace.TraceInfo(this.ToShortString() + ":{0} Aborting working thread as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers); if (workingThread != null) { workingThread.Abort(); } } else { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Continuing heavy part as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers); } } } if (stopEvent.WaitOne(TimeSpan.FromSeconds(settings.TouchPeriodInSeconds))) { JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} detected touch thread stop request. stopping touch thread loop...", job); break; } } catch (InvalidOperationException exc) { jobDeleted = true; JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} InvalidOperationException. Is job part deleted? ({1})", job, exc.ToString()); break; } catch (ChangeConflictException) { lock (jobsDataContext) { if (TraceConflicts(jobsDataContext, "last chance exc") || jobDeleted) { break; } } } catch (Exception exc) { JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Error updating touch time for job {1}", job, exc.ToString()); if (fails++ > 3) { break; } } if (jobDeleted) { JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} record is deleted. stoping touch thread", job); } //curPhys = currentProcess.WorkingSet64; //curPaged = currentProcess.PagedMemorySize64; //curVirtual = currentProcess.VirtualMemorySize64; //if (currentLogRecord.PeakPhysicalMemoryUsage < curPhys) currentLogRecord.PeakPhysicalMemoryUsage = curPhys; //if (currentLogRecord.PeakPagedMemoryUsage < curPaged) currentLogRecord.PeakPagedMemoryUsage = curPaged; //if (currentLogRecord.PeakVirtualMemoryUsage < curVirtual) currentLogRecord.PeakVirtualMemoryUsage = curVirtual; } JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread ending. Peak mem stats: working set {0}Mb, GC.allocated {1}Mb, PrivateMem {2}Mb, min available Ram {3}Mb", ((double)maxWorkingSet) / (1024 * 1024), ((double)maxGCTotal) / (1024 * 1024), ((double)maxPrivate) / (1024 * 1024), minRamAvailMb); //currentLogRecord.WorkEnd = DateTime.UtcNow; //logManager.Insert(currentLogRecord); }
public JobStatus Submit(IFetchRequest request, string hash, double jobRegistrationPermitedTime, int minPointPerPartition, int maxPointPerPartition, int totalWorkers) { var jobStatus = GetStatus(hash, false); int retries = 0; while (true) { if (jobStatus == null) { JobManagerTrace.TraceVerbose("{0}: Request is new (not in jobs table)", hash); // Try to create blob try { if (CreateBlobWithJob(request, hash, jobRegistrationPermitedTime)) // Successfully created { DateTime currentUTC = DateTime.UtcNow; int partsCount = GetPartitionsCount(request, minPointPerPartition, maxPointPerPartition, GetFreeWorkersNo(totalWorkers)); var partsToInsert = Enumerable.Range(0, partsCount).Select(partNumber => new Job() { Hash = hash, PartNo = partNumber, PartsCount = partsCount, Status = (byte)JobOrPartState.Pending, Priority = 0, SubmitTime = currentUTC, Touchtime = currentUTC }).ToArray(); try { jobsDataContext.Jobs.InsertAllOnSubmit(partsToInsert); jobsDataContext.SubmitChanges(System.Data.Linq.ConflictMode.FailOnFirstConflict); } catch (Exception ex) { JobManagerTrace.TraceError("{0}:Exception during insertion of job records into the job table. Reverting job registration (deleting job blob, all job records) {1}", hash, ex.ToString()); DiscardDataContextPendingChanges(); jobsDataContext.ExecuteCommand("delete from Job where hash='{0}'", hash); //order is important. deleting records before blob! DeleteJobBlob(hash); throw; //to be caught by retrying code } for (int i = 1; i <= partsCount; ++i) { JobManagerTrace.TraceVerbose("{0}:{1}:{2}:Record created in the jobs table", hash, i, partsCount); } return(GetStatus(hash)); } else // Blob exists { jobStatus = JobStatus.GetPendingStatus(hash, Int32.MaxValue); // Unknown queue length JobManagerTrace.TraceVerbose("{0}: Blob for request is created recently by someone else. Returning {1} status.", hash, jobStatus); return(jobStatus); } } catch (Exception e) { JobManagerTrace.TraceError("{0}:Request submitting error: {1}", hash, e.ToString()); if (retries++ > 3) { JobManagerTrace.TraceError("{0}:3 retries failed for request. Reporting failed state to the client", hash); return(JobStatus.GetFailedStatus(e.ToString())); } } } else { JobManagerTrace.TraceVerbose("{0}: Request is already submitted. Current state is {1}", hash, jobStatus); return(jobStatus); } } }