internal QueueAbstract GetQueue(XElement ep, Progress <QueueMonitorMessage> monitorPipelineProgress, string inputQueueName)
{
QueueAbstract queue = GetQueue(ep, monitorPipelineProgress);
queue.pipeInputQueueName = inputQueueName;
return(queue);
}
public FilterContextContains(XElement config, QueueAbstract queue) : base(config)
{
_contextCache = new MemoryCache(queue.queueName);
try
{
contextCacheKeyXPath = config.Attribute("contextCacheKeyXPath").Value;
}
catch (Exception)
{
contextCacheKeyXPath = null;
}
try
{
this.contextCacheExpiry = double.Parse(config.Attribute("contextCacheExpiry").Value);
}
catch (Exception)
{
this.contextCacheExpiry = 10.0;
}
try
{
this.useMessageAsKey = bool.Parse(config.Attribute("useMessageAsKey").Value);
}
catch (Exception)
{
this.useMessageAsKey = false;
}
// this.queue = queue;
}
public IQueueFilter GetFilter(XElement ep, QueueAbstract queue)
{
string type = ep.Name.ToString();
IQueueFilter filter = null;
switch (type)
{
case "xpexists":
filter = new FilterXPathExists(ep);
break;
case "xpequals":
case "xpmatches":
filter = new FilterXPathEquals(ep);
break;
case "dateRange":
filter = new FilterDateRange(ep);
break;
case "bool":
filter = new FilterBool(ep);
break;
case "contains":
filter = new FilterContains(ep);
break;
case "equals":
filter = new FilterEquals(ep);
break;
case "matches":
filter = new FilterMatches(ep);
break;
case "length":
filter = new FilterLength(ep);
break;
case "contextContains":
filter = new FilterContextContains(ep, queue);
break;
}
return(filter);
}
public Expression(XElement defn, QueueAbstract queue)
{
try
{
type = defn.Attribute("type").Value;
}
catch (Exception)
{
type = "single";
}
type = defn.Name.ToString();
foreach (string fType in filterTypes)
{
foreach (XElement filterConfig in defn.Elements(fType))
{
IQueueFilter filter = filterFact.GetFilter(filterConfig, queue);
if (filter != null)
{
filters.Add(filter);
}
// If it is a "single" type of expression, there will only be one filter Element
if (type == "single" || type == "not")
{
singleFilter = filter;
}
}
}
// Boolean operators are all implemented in the this class
foreach (string eType in expressionTypes)
{
foreach (XElement exprConfig in defn.Elements(eType))
{
Expression exp = new Expression(exprConfig, queue);
expressions.Add(exp);
if (type == "single" || type == "not")
{
singleExpression = exp;
}
}
}
}
private async Task <ExchangeMessage> SendAndLog(QueueAbstract queue, ExchangeMessage message)
{
message = await queue.Send(message);
return(message);
}
public Pipeline(XElement pipeConfig, IProgress <PipelineMonitorMessage> monitorMessageProgress)
{
this.monitorMessageProgress = monitorMessageProgress;
monitorPipelineProgress = new Progress <QueueMonitorMessage>();
monitorPipelineProgress.ProgressChanged += MonitorStatusMessage;
try {
string dist = pipeConfig.Attribute("distribution").Value;
if (dist == "random")
{
randomDistribution = true;
}
else if (dist == "roundRobin")
{
roundRobinDistribution = true;
}
} catch {
randomDistribution = false;
roundRobinDistribution = false;
}
try {
this.id = pipeConfig.Attribute("id").Value;
} catch (Exception) {
this.id = Guid.NewGuid().ToString();
}
try {
outputIsolation = bool.Parse(pipeConfig.Attribute("outputIsolation").Value);
} catch (Exception) {
outputIsolation = false;
}
try {
contextAware = bool.Parse(pipeConfig.Attribute("contextAware").Value);
} catch (Exception) {
contextAware = false;
}
try {
mostRecentOnly = bool.Parse(pipeConfig.Attribute("mostRecentOnly").Value);
} catch (Exception) {
mostRecentOnly = false;
}
try {
name = pipeConfig.Attribute("name").Value;
} catch (Exception) {
name = "Un Named PipeLine";
}
try {
inputQueueName = pipeConfig.Attribute("pipeInputQueueName").Value;
} catch (Exception) {
inputQueueName = null;
}
// Configure a throttling time to limit the throughput of the pipeline
try {
maxMsgPerMinute = int.Parse(pipeConfig.Attribute("maxMsgPerMinute").Value);
if (maxMsgPerMinute == -1)
{
throttleInterval = 0;
}
else
{
throttleInterval = 60000 / maxMsgPerMinute;
}
} catch (Exception) {
throttleInterval = 0;
}
// Configure a throttling time to limit the throughput of the pipeline
try {
maxMsgPerMinuteProfile = pipeConfig.Attribute("maxMsgPerMinuteProfile").Value;
} catch (Exception) {
maxMsgPerMinuteProfile = null;
}
// QueueFactory takes the definition of each of the queues and creates the defined
// queue of the defined type. All the filters and transformations are built into the
// queue itself and configured in the constructor
// The input queue. There may be multiple queue which have to be prioritised
IEnumerable <XElement> InEndPoints = from ep in pipeConfig.Descendants("input") select ep;
foreach (XElement ep in InEndPoints)
{
QueueAbstract queue = queueFactory.GetQueue(ep, monitorPipelineProgress, inputQueueName);
//queue.SetParentPipe(this);
if (queue != null)
{
if (!queue.isValid)
{
logger.Warn($"Could not add Input Queue {queue.queueName} to {name}. Invalid Configuration");
continue;
}
input.Add(queue);
}
else
{
logger.Error($"Could not proccess Input Queue {queue.queueName}");
}
}
// The output queues. There may be multiple output queues for each input queue
IEnumerable <XElement> OutEndPoints = from ep in pipeConfig.Descendants("output") select ep;
foreach (XElement ep in OutEndPoints)
{
try {
QueueAbstract queue = queueFactory.GetQueue(ep, monitorPipelineProgress);
if (queue != null)
{
if (!queue.isValid)
{
logger.Warn($"Could not add Output Queue {queue.queueName} to {name}. Invalid Configuration");
continue;
}
output.Add(queue);
}
else
{
logger.Error($"Could not proccess Output Queue {queue.queueName}");
}
} catch (Exception ex) {
logger.Error(ex.Message);
}
}
// Create a queue that all the inputs will sent messages to
try {
if (!MessageQueue.Exists(inputQueueName))
{
using (MessageQueue t = MessageQueue.Create(inputQueueName)) {
this.pipeInputQueue = t;
logger.Info($"Created Input Queue {inputQueueName} for pipeline");
}
}
else
{
this.pipeInputQueue = new MessageQueue(inputQueueName);
}
} catch (Exception ex) {
logger.Error(ex.Message);
}
_contextCache = new MemoryCache(name);
try {
contextCacheKeyXPath = pipeConfig.Attribute("contextCacheKeyXPath").Value;
} catch (Exception) {
contextCacheKeyXPath = null;
}
try {
this.contextCacheExpiry = double.Parse(pipeConfig.Attribute("contextCacheExpiry").Value);
} catch (Exception) {
this.contextCacheExpiry = 10.0;
}
try {
this.discardInCache = bool.Parse(pipeConfig.Attribute("discardInCache").Value);
} catch (Exception) {
this.discardInCache = false;
}
try {
this.firstOnly = bool.Parse(pipeConfig.Attribute("firstOnly").Value);
} catch (Exception) {
this.firstOnly = false;
}
try {
this.useMessageAsKey = bool.Parse(pipeConfig.Attribute("useMessageAsKey").Value);
} catch (Exception) {
this.useMessageAsKey = false;
}
//Output and reset context
if (contextCacheKeyXPath != null)
{
int contextStatsInterval;
try {
contextStatsInterval = int.Parse(pipeConfig.Attribute("contextStatsInterval").Value);
} catch (Exception) {
contextStatsInterval = 30000;
}
resetTimer = new System.Timers.Timer()
{
AutoReset = true,
Interval = contextStatsInterval
};
logger.Info($"Context Cache Stats Interval set to {contextStatsInterval}");
resetTimer.Elapsed += (source, eventArgs) =>
{
statLogger.Info($"Total received: {this.totalReceived}, Total sent: {this.totalSent}");
statLogger.Info($">>>> Context Cache Stats for Previous {resetTimer.Interval}ms");
try {
foreach (var v in statDict.Values)
{
statLogger.Info(v);
}
} catch (Exception ex) {
logger.Error($"Stats dictionary problem {ex.Message}");
} finally {
statDict.Clear();
}
statLogger.Info($"<<<<< End of Context Cache Stats");
};
resetTimer.Start();
}
/* Only allow if there are output queues configured
* It is OK if there are no inputs defined. An output only queue may be
* defined just to specifiy a maxMessages parameter on the queue for size maintenance
*/
if (output.Count() > 0)
{
OK_TO_RUN = true;
}
else
{
OK_TO_RUN = false;
logger.Warn($"No inputs or outputs defined for Pipeline {name}");
}
}
public QueueAbstract(XElement defn, IProgress <QueueMonitorMessage> monitorMessageProgress)
{
this.monitorMessageProgress = monitorMessageProgress;
this.definition = defn;
try {
var stylesheet = definition.Attribute("stylesheet").Value;
this.styleSheets = stylesheet.Split(',').ToList <string>();
this.bTransform = true;
}
catch (Exception) {
this.styleSheets = new List <string>();
this.bTransform = false;
}
try {
this.queueName = definition.Attribute("queue").Value;
}
catch (Exception) {
this.queueName = "undefined";
}
try {
this.id = definition.Attribute("id").Value;
}
catch (Exception) {
this.id = Guid.NewGuid().ToString();
}
try {
this.name = definition.Attribute("name").Value;
}
catch (Exception) {
this.name = this.queueName;
}
try {
this.createQueue = bool.Parse(definition.Attribute("createQueue").Value);
}
catch (Exception) {
this.createQueue = false;
}
try {
this.xslVersion = definition.Attribute("xslVersion").Value;
}
catch (Exception) {
this.xslVersion = "1.0";
}
try {
this.maxRetry = Convert.ToInt32(definition.Attribute("maxRetry").Value);
}
catch (Exception) {
this.maxRetry = 10;
}
try {
this.priority = Convert.ToInt32(definition.Attribute("priority").Value);
switch (priority)
{
case 0:
mqPriority = MessagePriority.Lowest;
break;
case 1:
mqPriority = MessagePriority.VeryLow;
break;
case 2:
mqPriority = MessagePriority.Low;
break;
case 3:
mqPriority = MessagePriority.Normal;
break;
case 4:
mqPriority = MessagePriority.AboveNormal;
break;
case 5:
mqPriority = MessagePriority.High;
break;
case 6:
mqPriority = MessagePriority.VeryHigh;
break;
case 7:
mqPriority = MessagePriority.Highest;
break;
}
}
catch (Exception) {
this.priority = 2;
mqPriority = MessagePriority.Normal;
}
try {
undeliverableQueue = defn.Attribute("undeliverableQueue").Value;
CreateQueue(undeliverableQueue);
}
catch (Exception) {
undeliverableQueue = null;
}
try {
bufferQueueName = definition.Attribute("bufferQueueName").Value;
CreateQueue(bufferQueueName, true);
}
catch (Exception) {
bufferQueueName = null;
}
try {
xpathDestination = definition.Attribute("xpathDestination").Value;
}
catch (Exception) {
xpathDestination = null;
}
try {
xpathContentDestination = definition.Attribute("xpathContentDestination").Value;
}
catch (Exception) {
xpathContentDestination = null;
}
// Add any filter expresion
XElement filtersDefn = defn.Element("filter");
if (filtersDefn != null)
{
// Add the alternate queue to send to if the expression fails (optional)
try {
altQueue = fact.GetQueue(filtersDefn.Element("altqueue"), this.monitorMessageProgress);
}
catch (Exception) {
altQueue = null;
}
/*
* At the top level, there is only one Expresssion, which it self can be a compound
* expression or a single data filter
*
* When the Expression itself is constucted, it recurssive creates all the Expression o
* filters configured under it
*/
// Cycle through the expressions types (and, or, not, xor) to see if any exist
foreach (string eType in Expression.expressionTypes)
{
XElement exprDefn = filtersDefn.Element(eType);
if (exprDefn != null)
{
expression = new Expression(exprDefn, this);
}
}
// Cycle through the data filter types (and, or, not, xor) to see if any exist
foreach (string fType in Expression.filterTypes)
{
XElement filtDefn = filtersDefn.Element(fType);
if (filtDefn != null)
{
topLevelFilter = fact.GetFilter(filtDefn, this);
}
}
}
// Call the instance specific setup
this.isValid = this.SetUp();
}
