public void Start()
{
if (_started)
{
return;
}
var options = new ParallelOptions
{
MaxDegreeOfParallelism = _config.MaxDegreeOfParallelism.HasValue ? _config.MaxDegreeOfParallelism.Value : 10,
CancellationToken = _cancel.Token
};
if (_background != null)
{
Stop();
_background.Dispose();
_background = null;
}
ReseedQueueFromBacklog();
_background = Task.Factory.StartNew(() =>
{
try
{
if (_config.MaxDeliveryRate.HasValue)
{
Parallel.ForEach(_outgoing.GetConsumingPartitioner(), options, (message, state) =>
{
while (DeliveryRate > _config.MaxDeliveryRate)
{
Thread.Sleep(10);
}
DeliveryCycle(options, message, state);
});
}
else
{
Parallel.ForEach(_outgoing.GetConsumingPartitioner(), options, (message, state) => DeliveryCycle(options, message, state));
}
}
catch (OperationCanceledException)
{
while (!_outgoing.IsCompleted)
{
_backlog.Enqueue(_outgoing.Take());
}
}
});
_uptime.Start();
_started = true;
}
public override void PurgeEntities(out int numEntitiesProcessed, out int numPartitionsProcessed)
{
VerifyIsInitialized();
void CollectData()
{
CollectDataToProcess(PurgeEntitiesOlderThanDays);
}
void ProcessData()
{
Parallel.ForEach(_partitionKeyQueue.GetConsumingPartitioner(), new ParallelOptions {
MaxDegreeOfParallelism = MaxParallelOperations
}, ProcessPartition);
}
_cancellationTokenSource = new CancellationTokenSource();
Parallel.Invoke(new ParallelOptions {
CancellationToken = _cancellationTokenSource.Token
}, CollectData, ProcessData);
numPartitionsProcessed = _globalPartitionCounter;
numEntitiesProcessed = _globalEntityCounter;
}
private void ProduceOn(BlockingCollection <T> source, ParallelOptions options)
{
var partitioner = source.GetConsumingPartitioner();
Parallel.ForEach(partitioner, options,
async(@event, state) => await ProductionCycle(options, @event, state));
}
void Run()
{
//_systemLogger.Trace($"Is ThreadPoolThread {Thread.CurrentThread.IsThreadPoolThread}");
try
{
var res = Parallel.ForEach(_taskContextsQueue.GetConsumingPartitioner(),
new ParallelOptions {
CancellationToken = _processToken, MaxDegreeOfParallelism = _maxConsumers
},
ExecuteTask);
//_systemLogger.Trace($"Is ThreadPool Thread {Thread.CurrentThread.IsThreadPoolThread}");
_logger.Trace(
$"Consumer process {_processId} stopped gracefully with completion flag: {res.IsCompleted}");
}
catch (TaskCanceledException e)
{
var msg =
$"Consumer canceled for Process: {_processKey} by token {(_processToken.IsCancellationRequested ? "ProcessToken" : (_parentToken.IsCancellationRequested ? "ParentToken" : "NoToken"))} with msg {e.Message}";
_systemLogger.Info(msg);
}
catch (OperationCanceledException e)
{
var msg =
$"Consumer canceled for Process: {_processKey} by token {(_processToken.IsCancellationRequested ? "ProcessToken" : (_parentToken.IsCancellationRequested ? "ParentToken" : "NoToken"))} with msg {e.Message}";
_systemLogger.Info(msg);
}
catch (AggregateException e)
{
//probably thread abort exception
_systemLogger.Warn(
$"Consumer got exception while processing Process: {_processKey} with message {e.InnerException?.Message ?? string.Empty}",
e);
//e.Handle(ex =>
foreach (var ex in e.InnerExceptions)
{
if (ex is ThreadAbortException)
{
Robustness.Instance.SafeCall(Thread.ResetAbort);
}
//return true;
} //);
}
catch (ThreadAbortException)
{
_systemLogger.Warn(
$"Consumer got ThreadAbortException while processing Process: {_processKey} with message");
Robustness.Instance.SafeCall(Thread.ResetAbort);
}
catch (Exception e)
{
_logger.Error($"Consumer stopped unexpectedly Process: {_processKey} with message {e.Message}", e);
}
//finally
//{
// _systemLogger.Trace($"Consumer final for Process: {_processKey}");
//}
}
public void Start(TimeSpan lockExpiry, CancellationToken cancellationToken)
{
if (_pipeInfos != null)
{
throw new ApplicationException("Reader instance already started");
}
_pipeInfos = new BlockingCollection <PipeInfo>();
_taskFunnel.ListenForPipeEvents(/*_taskExecutors.Keys, */ _pipeInfos);
ExecuteExisting(lockExpiry);
Parallel.ForEach(_pipeInfos.GetConsumingPartitioner(cancellationToken), new ParallelOptions()
{
MaxDegreeOfParallelism = _maxThreads, CancellationToken = cancellationToken
}, pipeInfo =>
{
var taskExecutor = _taskExecutors[pipeInfo.ParentPipeName];
var messageBatch = _taskFunnel.TryReadMessageBatch(true, pipeInfo, lockExpiry, 2);
ExecuteBatch(taskExecutor, pipeInfo, messageBatch);
});
}
/// <summary>
/// Reads, parses, and inserts all of the Cities from the Geonames city files
/// found on the provided directory. The provided directory must contain
/// Valid Geonames city files.
/// </summary>
public void ReadCityFiles()
{
Parallel.ForEach <GeonamesFileInfo>(
_geonamesFiles.GetConsumingPartitioner <GeonamesFileInfo>(),
new ParallelOptions {
MaxDegreeOfParallelism = _maxParallelism
},
(inputFile, args) =>
{
RaiseFileParsing(inputFile);
using (StreamReader input = new StreamReader(inputFile.FullName))
{
while (!input.EndOfStream)
{
RaiseEntryParsed(ParseCity(input.ReadLine()));
Interlocked.Increment(ref _parsedEntries);
}
}
RaiseFileParsed(inputFile);
});
RaiseDirectoryParsed(Directory);
}
private void InitializeConsumers()
{
#if DEBUG
if (!HangFireScheduler._stopWatch.IsRunning)
{
HangFireScheduler._stopWatch.Start();
}
#endif
Task.Run(() =>
{
Parallel.ForEach(_validatorsToRun.GetConsumingPartitioner(), validatorInfo =>
{
IList <IValidatorRunEntry> results = this.RunValidatorInstance(validatorInfo);
ValidatorResultsReady.Invoke(this, new ValidatorResultsArg(results));
#if DEBUG
var timeElapsed = HangFireScheduler._stopWatch.Elapsed;
JobsRan++;
Console.WriteLine(string.Format("Time Elapsed {5} Handled by Thread {4} - Jobs Queued: {0} Jobs Ran {1} Jobs Scheduled: {2} Jobs Triggered: {3}", JobsQueued, JobsRan, HangFireScheduler.JobsScheduled, HangFireScheduler.JobsScheduledCalledBack, Thread.CurrentThread.ManagedThreadId, timeElapsed));
#endif
})
;
});
}