/// <summary>Marks all requests that are not updated for a specified time as failed</summary>
/// <param name="permittedUntouchedIntervalSec">Maximum allowed idle time in seconds</param>
public void MarkOutdatedAsFailed(int permittedUntouchedIntervalSec)
{
DateTime currentUTC = DateTime.UtcNow;
var recyledParts = (from job in jobsDataContext.Jobs
where job.Status == (byte)JobOrPartState.InProgress && (currentUTC - job.Touchtime).TotalSeconds > permittedUntouchedIntervalSec
select job).ToArray();
if (recyledParts.Length > 0)
{
lock (jobsDataContext)
{
foreach (var j in recyledParts)
{
JobManagerTrace.TraceError("{0}:{1}:{2}:TouchTime hasn't been updated for too long. Marking it as failed", j.Hash, j.PartNo, j.PartsCount);
j.Status = (byte)JobOrPartState.Failed;
}
try
{
jobsDataContext.Refresh(RefreshMode.KeepChanges, recyledParts);
jobsDataContext.SubmitChanges(ConflictMode.ContinueOnConflict);
}
catch (ChangeConflictException)
{
lock (jobsDataContext)
{
jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.KeepChanges);
jobsDataContext.SubmitChanges(ConflictMode.FailOnFirstConflict);
}
}
}
}
}
private void CompleteWithStatus(byte status)
{
lock (jobsDataContext)
{
try
{
//Trace.WriteLine(string.Format("setting job {0} status {1}",this,status));
jobsDataContext.Refresh(RefreshMode.KeepChanges, job);
job.Status = status;
jobsDataContext.SubmitChanges();
}
catch (ChangeConflictException)
{
foreach (var item in jobsDataContext.ChangeConflicts)
{
if (item.IsDeleted)
{
JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":The job was unexpectedly deleted");
}
foreach (var conflict in item.MemberConflicts)
{
JobManager.JobManagerTrace.TraceError(this.ToShortString() + ":{4} Conflict {3} original:{0} current:{1} database:{2}", conflict.OriginalValue, conflict.CurrentValue, conflict.DatabaseValue, conflict.Member.Name, this);
}
}
throw;
}
finally
{
stopTouch.Set();
}
}
}
private void TouchThreadFunc(object state)
{
Job job = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item1;
JobsDBDataContext jobsDataContext = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item2;
ManualResetEvent stopEvent = ((Tuple <Job, JobsDBDataContext, ManualResetEvent>)state).Item3;
var ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes");
long curWorkingSet, curGCTotal, curPrivate;
long maxWorkingSet = 0;
long maxGCTotal = 0;
long maxPrivate = 0;
double curRamAvailMb, minRamAvailMb = double.MaxValue;
int fails = 0;
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread running");
bool jobDeleted = false;
while (true)
{
Process currentProcess = Process.GetCurrentProcess();
currentProcess.Refresh();
curWorkingSet = Environment.WorkingSet;
curGCTotal = GC.GetTotalMemory(false);
curPrivate = currentProcess.PrivateMemorySize64;
curRamAvailMb = ramCounter.NextValue();
if (maxWorkingSet < curWorkingSet)
{
maxWorkingSet = curWorkingSet;
}
if (maxGCTotal < curGCTotal)
{
maxGCTotal = curGCTotal;
}
if (maxPrivate < curPrivate)
{
maxPrivate = curPrivate;
}
if (minRamAvailMb > curRamAvailMb)
{
minRamAvailMb = curRamAvailMb;
}
try
{
bool cancelationCheckNeeded = false;
lock (jobsDataContext)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Touch thread tick", job);
bool touched = false;
for (int touchTries = 0; touchTries < 5; touchTries++)
{
try
{
jobsDataContext.Refresh(RefreshMode.KeepChanges, job);
job.Touchtime = DateTime.UtcNow;
jobsDataContext.SubmitChanges();
touched = true;
break;
}
catch (ChangeConflictException)
{
//TraceConflicts(jobsDataContext, "updating touch time");
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to update touchtime. attempt {1}", job, touchTries + 1);
jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues);
}
}
if (touched)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Updated TouchTime", job);
}
else
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Failed to update TouchTime", job);
}
if (job.IsHeavyJob == 0)
{
var execTimespanSec = (DateTime.UtcNow - job.StartTime.Value).TotalSeconds;
if (execTimespanSec > settings.LightJobExecutionPermitedTimeSec)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Job considered to be HEAVY", job);
for (int retries = 0; retries < 10; retries++)
{
try
{
var allParts = jobsDataContext.Jobs.Where(j1 => j1.Hash == job.Hash).ToArray();
foreach (var j in allParts)
{
j.IsHeavyJob = 1;
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Committing HEAVY flag for all jobs part", job);
jobsDataContext.Refresh(RefreshMode.KeepChanges, allParts);
jobsDataContext.SubmitChanges(ConflictMode.ContinueOnConflict);
break;
}
catch (ChangeConflictException)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Failed to set heavy flag for all parts. attempt {1}", job, retries + 1);
//TraceConflicts(jobsDataContext, "heavy flag committing");
jobsDataContext.ChangeConflicts.ResolveAll(RefreshMode.OverwriteCurrentValues);
}
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Successfully committed HEAVY flag for all jobs part", job);
cancelationCheckNeeded = true;
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job still light ({1} execution seconds of {2} permitted)", job, execTimespanSec, settings.LightJobExecutionPermitedTimeSec);
}
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} touched job part record. Job is already marked as heavy", job);
cancelationCheckNeeded = true;
}
if (cancelationCheckNeeded)
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Checking for heavy job cancelation", job);
int otherRunningHeavyPartsCount = jobsDataContext.Jobs.Count(j2 => j2.StartTime < job.StartTime && j2.Status == 1 && j2.IsHeavyJob == 1);
if (otherRunningHeavyPartsCount >= settings.PermitedHeavyPartWorkers)
{
JobManager.JobManagerTrace.TraceInfo(this.ToShortString() + ":{0} Aborting working thread as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers);
if (workingThread != null)
{
workingThread.Abort();
}
}
else
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} Continuing heavy part as there are {1} other active heavy parts ({2} total permitted)", job, otherRunningHeavyPartsCount, settings.PermitedHeavyPartWorkers);
}
}
}
if (stopEvent.WaitOne(TimeSpan.FromSeconds(settings.TouchPeriodInSeconds)))
{
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":{0} detected touch thread stop request. stopping touch thread loop...", job);
break;
}
}
catch (InvalidOperationException exc)
{
jobDeleted = true;
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} InvalidOperationException. Is job part deleted? ({1})", job, exc.ToString());
break;
}
catch (ChangeConflictException)
{
lock (jobsDataContext)
{
if (TraceConflicts(jobsDataContext, "last chance exc") || jobDeleted)
{
break;
}
}
}
catch (Exception exc)
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} Error updating touch time for job {1}", job, exc.ToString());
if (fails++ > 3)
{
break;
}
}
if (jobDeleted)
{
JobManager.JobManagerTrace.TraceWarning(this.ToShortString() + ":{0} record is deleted. stoping touch thread", job);
}
//curPhys = currentProcess.WorkingSet64;
//curPaged = currentProcess.PagedMemorySize64;
//curVirtual = currentProcess.VirtualMemorySize64;
//if (currentLogRecord.PeakPhysicalMemoryUsage < curPhys) currentLogRecord.PeakPhysicalMemoryUsage = curPhys;
//if (currentLogRecord.PeakPagedMemoryUsage < curPaged) currentLogRecord.PeakPagedMemoryUsage = curPaged;
//if (currentLogRecord.PeakVirtualMemoryUsage < curVirtual) currentLogRecord.PeakVirtualMemoryUsage = curVirtual;
}
JobManager.JobManagerTrace.TraceVerbose(this.ToShortString() + ":Touch thread ending. Peak mem stats: working set {0}Mb, GC.allocated {1}Mb, PrivateMem {2}Mb, min available Ram {3}Mb",
((double)maxWorkingSet) / (1024 * 1024), ((double)maxGCTotal) / (1024 * 1024), ((double)maxPrivate) / (1024 * 1024), minRamAvailMb);
//currentLogRecord.WorkEnd = DateTime.UtcNow;
//logManager.Insert(currentLogRecord);
}