/// <summary>
/// The actual delegate
/// </summary>
/// <param name="processor"></param>
/// <param name="queueItem"></param>
private void ExecuteProcessor(IWorkQueueItemProcessor processor, Model.WorkQueue queueItem)
{
try
{
processor.Process(queueItem);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e,
"Unexpected exception when processing WorkQueue item of type {0}. Failing Queue item. (GUID: {1})",
queueItem.WorkQueueTypeEnum,
queueItem.GetKey());
String error = e.InnerException != null ? e.InnerException.Message : e.Message;
FailQueueItem(queueItem, error);
}
finally
{
// Signal the parent thread, so it can query again
_threadStop.Set();
// Cleanup the processor
processor.Dispose();
}
}
/// <summary>
/// Enqueue a WorkQueue entry for processing.
/// </summary>
/// <param name="processor"></param>
/// <param name="item"></param>
/// <param name="del"></param>
public void Enqueue(IWorkQueueItemProcessor processor, Model.WorkQueue item, WorkQueueThreadDelegate del)
{
WorkQueueThreadParameter parameter = new WorkQueueThreadParameter(processor, item, del);
lock (_syncLock)
{
if (item.WorkQueuePriorityEnum.Equals(WorkQueuePriorityEnum.High))
{
_highPriorityCount++;
}
if (item.WorkQueuePriorityEnum.Equals(WorkQueuePriorityEnum.Stat))
{
_highPriorityCount++;
}
WorkQueueTypeProperties prop = _workQueuePropList[item.WorkQueueTypeEnum];
if (prop.MemoryLimited)
{
_memoryLimitedCount++;
}
_totalThreadCount++;
_queuedItems.Add(parameter);
}
Enqueue(parameter, delegate(WorkQueueThreadParameter threadParameter)
{
threadParameter.Delegate(threadParameter.Processor, threadParameter.Item);
QueueItemComplete(threadParameter.Item);
});
}
/// <summary>
/// The processing thread.
/// </summary>
/// <remarks>
/// This method queries the database for WorkQueue entries to work on, and then uses
/// a thread pool to process the entries.
/// </remarks>
public void Run()
{
// Force the alert to be displayed right away, if it happens
DateTime lastLog = Platform.Time.AddMinutes(-61);
if (!_threadPool.Active)
{
_threadPool.Start();
}
Platform.Log(LogLevel.Info, "Work Queue Processor running...");
while (true)
{
if (_stop)
{
return;
}
bool threadsAvailable = _threadPool.CanQueueItem;
bool memoryAvailable = WorkQueueSettings.Instance.WorkQueueMinimumFreeMemoryMB == 0
||
SystemResources.GetAvailableMemory(SizeUnits.Megabytes) >
WorkQueueSettings.Instance.WorkQueueMinimumFreeMemoryMB;
if (threadsAvailable && memoryAvailable)
{
try
{
Model.WorkQueue queueListItem = GetWorkQueueItem(ServerPlatform.ProcessorId);
if (queueListItem == null)
{
/* No result found, or reach max queue entries for each type */
_terminateEvent.WaitOne(WorkQueueSettings.Instance.WorkQueueQueryDelay, false);
continue;
}
if (!_extensions.ContainsKey(queueListItem.WorkQueueTypeEnum))
{
Platform.Log(LogLevel.Error,
"No extensions loaded for WorkQueue item type: {0}. Failing item.",
queueListItem.WorkQueueTypeEnum);
//Just fail the WorkQueue item, not much else we can do
FailQueueItem(queueListItem, "No plugin to handle WorkQueue type: " + queueListItem.WorkQueueTypeEnum);
continue;
}
try
{
IWorkQueueProcessorFactory factory = _extensions[queueListItem.WorkQueueTypeEnum];
IWorkQueueItemProcessor processor = factory.GetItemProcessor();
// Enqueue the actual processing of the item to the thread pool.
_threadPool.Enqueue(processor, queueListItem, ExecuteProcessor);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception creating WorkQueue processor.");
FailQueueItem(queueListItem, "Failure getting WorkQueue processor: " + e.Message);
continue;
}
}
catch (Exception e)
{
// Wait for only 1.5 seconds
Platform.Log(LogLevel.Error, e, "Exception occured when processing WorkQueue item.");
_terminateEvent.WaitOne(3000, false);
}
}
else
{
if ((lastLog.AddMinutes(60) < Platform.Time) && !memoryAvailable)
{
lastLog = Platform.Time;
Platform.Log(LogLevel.Error, "Unable to process WorkQueue entries, Minimum memory not available, minimum MB required: {0}, current MB available:{1}",
WorkQueueSettings.Instance.WorkQueueMinimumFreeMemoryMB,
SystemResources.GetAvailableMemory(SizeUnits.Megabytes));
ServerPlatform.Alert(AlertCategory.Application, AlertLevel.Critical, "WorkQueue", AlertTypeCodes.NoResources,
null, TimeSpan.Zero,
"Unable to process WorkQueue entries, Minimum memory not available, minimum MB required: {0}, current MB available:{1}",
WorkQueueSettings.Instance.WorkQueueMinimumFreeMemoryMB,
SystemResources.GetAvailableMemory(SizeUnits.Megabytes));
}
// wait for new opening in the pool or termination
WaitHandle.WaitAny(new WaitHandle[] { _threadStop, _terminateEvent }, 3000, false);
_threadStop.Reset();
}
}
}
public WorkQueueThreadParameter(IWorkQueueItemProcessor processor, Model.WorkQueue item, WorkQueueThreadDelegate del)
{
_item = item;
_processor = processor;
_del = del;
}
/// <summary>
/// The actual delegate
/// </summary>
/// <param name="processor"></param>
/// <param name="queueItem"></param>
private void ExecuteProcessor(IWorkQueueItemProcessor processor, Model.WorkQueue queueItem)
{
try
{
processor.Process(queueItem);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e,
"Unexpected exception when processing WorkQueue item of type {0}. Failing Queue item. (GUID: {1})",
queueItem.WorkQueueTypeEnum,
queueItem.GetKey());
String error = e.InnerException != null ? e.InnerException.Message : e.Message;
FailQueueItem(queueItem, error);
}
finally
{
// Signal the parent thread, so it can query again
_threadStop.Set();
// Cleanup the processor
processor.Dispose();
}
}
