private static string SafeGetParameter(QueuedJobDto nextJob, String parameter)
{
if (nextJob.Parameters.ContainsKey(parameter))
{
return(nextJob.Parameters[parameter]);
}
return(String.Empty);
}
private static PollerJobParameters ExtractJobParameters(QueuedJobDto nextJob)
{
PollerJobParameters parameters = new PollerJobParameters();
parameters.FileExtension = SafeGetParameter(nextJob, JobKeys.FileExtension);
parameters.FileName = SafeGetParameter(nextJob, JobKeys.FileName);
parameters.InputDocumentFormat = new DocumentFormat(SafeGetParameter(nextJob, JobKeys.Format));
parameters.JobId = nextJob.Id;
parameters.TenantId = SafeGetParameter(nextJob, JobKeys.TenantId);
parameters.All = nextJob.Parameters;
return(parameters);
}
private QueuedJobDto DsGetNextJob()
{
try
{
QueuedJobDto nextJob = null;
string pollerResult;
using (WebClientEx client = new WebClientEx())
{
//TODO: use round robin if a document store is down.
var firstUrl = _dsEndpoints.First();
var payload = JsonConvert.SerializeObject(new
{
QueueName = this.QueueName,
Identity = this._identity,
Handle = this._handle,
});
Logger.DebugFormat("Polling url: {0} with payload {1}", firstUrl, payload);
client.Headers[HttpRequestHeader.ContentType] = "application/json";
pollerResult = client.UploadString(firstUrl.GetNextJobUrl, payload);
Logger.DebugFormat("GetNextJobResult: {0}", pollerResult);
}
if (!pollerResult.Equals("null", StringComparison.OrdinalIgnoreCase))
{
nextJob = JsonConvert.DeserializeObject <QueuedJobDto>(pollerResult, _settings);
}
_lastGoodCommunication = DateTime.Now;
return(nextJob);
}
catch (Exception ex)
{
if (DateTime.UtcNow.AddMinutes(-15) > _lastCommunicationError)
{
//new error in 15 minutes, we need to log
Logger.ErrorFormat(ex, "Unable to contact Document Store at address: {0}", _dsEndpoints.First());
_lastCommunicationError = DateTime.UtcNow;
}
else
{
Logger.InfoFormat("Document store cannot be reached, down since {0}", _lastGoodCommunication);
}
return(null);
}
}
private void ExecuteJobCore()
{
do
{
//if half of the task thread finished working, we should end all the pool and restart
if (ThreadNumber > 2 && _numOfPollerTaskActive < (ThreadNumber / 2))
{
//This can happen because jobs are generated not in block, if we have ex 6 threads
//base jobs started 6 tasks, then if in a moment only one task remain only one task remain active
//then if the queue manager queue 100 jobs, we have only one thread active. This condition
//stops the poll if half of the threads are active, so we need to restart polling with all the tasks.
return;
}
String workingFolder = null;
QueuedJobDto nextJob = DsGetNextJob();
if (nextJob == null)
{
System.Threading.Interlocked.Decrement(ref _numOfPollerTaskActive);
return;
}
Logger.ThreadProperties["job-id"] = nextJob.Id;
var baseParameters = ExtractJobParameters(nextJob);
//remember to enter the right tenant.
workingFolder = Path.Combine(
JobsHostConfiguration.GetWorkingFolder(baseParameters.TenantId, GetType().Name),
baseParameters.JobId
);
if (Directory.Exists(workingFolder))
{
Directory.Delete(workingFolder, true);
}
Directory.CreateDirectory(workingFolder);
try
{
var task = OnPolling(baseParameters, workingFolder);
var result = task.Result;
if (result.Result)
{
Logger.DebugFormat("Successfully executed Job: {0}", nextJob.Id);
}
else
{
Logger.ErrorFormat("Job {0} completed with errors: {1} with result", nextJob.Id, result.ErrorMessage);
}
//The execution if failed can be posticipated to future time, probably because the job can retry after a certain
//period of time.
if (!result.Posticipate)
{
DsSetJobExecuted(QueueName, nextJob.Id, result.ErrorMessage, result.ParametersToModify);
}
else
{
DsReQueueJob(QueueName, nextJob.Id, result.ErrorMessage, result.PosticipateExecutionTimestamp, result.ParametersToModify);
}
}
catch (AggregateException aex)
{
Logger.ErrorFormat(aex, "Error executing queued job {0} on tenant {1} - {2}",
nextJob.Id,
nextJob.Parameters[JobKeys.TenantId],
aex?.InnerExceptions?[0]?.Message);
StringBuilder aggregateMessage = new StringBuilder();
aggregateMessage.Append(aex.Message);
foreach (var ex in aex.InnerExceptions)
{
var errorMessage = String.Format("Inner error queued job {0} queue {1}: {2}", nextJob.Id, this.QueueName, ex.Message);
LogExceptionAndAllInnerExceptions(ex, errorMessage);
aggregateMessage.Append(errorMessage);
}
DsSetJobExecuted(QueueName, nextJob.Id, aggregateMessage.ToString(), null);
}
catch (Exception ex)
{
var errorMessage = String.Format("Error executing queued job {0} on tenant {1}", nextJob.Id, nextJob.Parameters[JobKeys.TenantId]);
LogExceptionAndAllInnerExceptions(ex, errorMessage);
DsSetJobExecuted(QueueName, nextJob.Id, ex.Message, null);
}
finally
{
DeleteWorkingFolder(workingFolder);
Logger.ThreadProperties["job-id"] = null;
}
} while (true); //Exit is in the internal loop
}
