Example #1
0
        internal QueueAbstract GetQueue(XElement ep, Progress <QueueMonitorMessage> monitorPipelineProgress, string inputQueueName)
        {
            QueueAbstract queue = GetQueue(ep, monitorPipelineProgress);

            queue.pipeInputQueueName = inputQueueName;
            return(queue);
        }
Example #2
0
        public FilterContextContains(XElement config, QueueAbstract queue) : base(config)
        {
            _contextCache = new MemoryCache(queue.queueName);

            try
            {
                contextCacheKeyXPath = config.Attribute("contextCacheKeyXPath").Value;
            }
            catch (Exception)
            {
                contextCacheKeyXPath = null;
            }

            try
            {
                this.contextCacheExpiry = double.Parse(config.Attribute("contextCacheExpiry").Value);
            }
            catch (Exception)
            {
                this.contextCacheExpiry = 10.0;
            }

            try
            {
                this.useMessageAsKey = bool.Parse(config.Attribute("useMessageAsKey").Value);
            }
            catch (Exception)
            {
                this.useMessageAsKey = false;
            }

            //          this.queue = queue;
        }
Example #3
0
        public IQueueFilter GetFilter(XElement ep, QueueAbstract queue)
        {
            string type = ep.Name.ToString();

            IQueueFilter filter = null;

            switch (type)
            {
            case "xpexists":
                filter = new FilterXPathExists(ep);
                break;

            case "xpequals":
            case "xpmatches":
                filter = new FilterXPathEquals(ep);
                break;

            case "dateRange":
                filter = new FilterDateRange(ep);
                break;

            case "bool":
                filter = new FilterBool(ep);
                break;

            case "contains":
                filter = new FilterContains(ep);
                break;

            case "equals":
                filter = new FilterEquals(ep);
                break;

            case "matches":
                filter = new FilterMatches(ep);
                break;

            case "length":
                filter = new FilterLength(ep);
                break;

            case "contextContains":
                filter = new FilterContextContains(ep, queue);
                break;
            }

            return(filter);
        }
Example #4
0
        public Expression(XElement defn, QueueAbstract queue)
        {
            try
            {
                type = defn.Attribute("type").Value;
            }
            catch (Exception)
            {
                type = "single";
            }

            type = defn.Name.ToString();

            foreach (string fType in filterTypes)
            {
                foreach (XElement filterConfig in defn.Elements(fType))
                {
                    IQueueFilter filter = filterFact.GetFilter(filterConfig, queue);
                    if (filter != null)
                    {
                        filters.Add(filter);
                    }

                    // If it is a "single" type of expression, there will only be one filter Element
                    if (type == "single" || type == "not")
                    {
                        singleFilter = filter;
                    }
                }
            }

            // Boolean operators are all implemented in the this class

            foreach (string eType in expressionTypes)
            {
                foreach (XElement exprConfig in defn.Elements(eType))
                {
                    Expression exp = new Expression(exprConfig, queue);
                    expressions.Add(exp);
                    if (type == "single" || type == "not")
                    {
                        singleExpression = exp;
                    }
                }
            }
        }
Example #5
0
        private async Task <ExchangeMessage> SendAndLog(QueueAbstract queue, ExchangeMessage message)
        {
            message = await queue.Send(message);

            return(message);
        }
Example #6
0
        public Pipeline(XElement pipeConfig, IProgress <PipelineMonitorMessage> monitorMessageProgress)
        {
            this.monitorMessageProgress              = monitorMessageProgress;
            monitorPipelineProgress                  = new Progress <QueueMonitorMessage>();
            monitorPipelineProgress.ProgressChanged += MonitorStatusMessage;

            try {
                string dist = pipeConfig.Attribute("distribution").Value;
                if (dist == "random")
                {
                    randomDistribution = true;
                }
                else if (dist == "roundRobin")
                {
                    roundRobinDistribution = true;
                }
            } catch {
                randomDistribution     = false;
                roundRobinDistribution = false;
            }

            try {
                this.id = pipeConfig.Attribute("id").Value;
            } catch (Exception) {
                this.id = Guid.NewGuid().ToString();
            }

            try {
                outputIsolation = bool.Parse(pipeConfig.Attribute("outputIsolation").Value);
            } catch (Exception) {
                outputIsolation = false;
            }

            try {
                contextAware = bool.Parse(pipeConfig.Attribute("contextAware").Value);
            } catch (Exception) {
                contextAware = false;
            }
            try {
                mostRecentOnly = bool.Parse(pipeConfig.Attribute("mostRecentOnly").Value);
            } catch (Exception) {
                mostRecentOnly = false;
            }

            try {
                name = pipeConfig.Attribute("name").Value;
            } catch (Exception) {
                name = "Un Named PipeLine";
            }

            try {
                inputQueueName = pipeConfig.Attribute("pipeInputQueueName").Value;
            } catch (Exception) {
                inputQueueName = null;
            }


            // Configure a throttling time to limit the throughput of the pipeline
            try {
                maxMsgPerMinute = int.Parse(pipeConfig.Attribute("maxMsgPerMinute").Value);
                if (maxMsgPerMinute == -1)
                {
                    throttleInterval = 0;
                }
                else
                {
                    throttleInterval = 60000 / maxMsgPerMinute;
                }
            } catch (Exception) {
                throttleInterval = 0;
            }

            // Configure a throttling time to limit the throughput of the pipeline
            try {
                maxMsgPerMinuteProfile = pipeConfig.Attribute("maxMsgPerMinuteProfile").Value;
            } catch (Exception) {
                maxMsgPerMinuteProfile = null;
            }
            // QueueFactory takes the definition of each of the queues and creates the defined
            // queue of the defined type. All the filters and transformations are built into the
            // queue itself and configured in the constructor

            // The input queue. There may be multiple queue which have to be prioritised
            IEnumerable <XElement> InEndPoints = from ep in pipeConfig.Descendants("input") select ep;

            foreach (XElement ep in InEndPoints)
            {
                QueueAbstract queue = queueFactory.GetQueue(ep, monitorPipelineProgress, inputQueueName);
                //queue.SetParentPipe(this);
                if (queue != null)
                {
                    if (!queue.isValid)
                    {
                        logger.Warn($"Could not add Input Queue {queue.queueName} to {name}. Invalid Configuration");
                        continue;
                    }
                    input.Add(queue);
                }
                else
                {
                    logger.Error($"Could not proccess Input Queue {queue.queueName}");
                }
            }


            // The output queues. There may be multiple output queues for each input queue
            IEnumerable <XElement> OutEndPoints = from ep in pipeConfig.Descendants("output") select ep;

            foreach (XElement ep in OutEndPoints)
            {
                try {
                    QueueAbstract queue = queueFactory.GetQueue(ep, monitorPipelineProgress);

                    if (queue != null)
                    {
                        if (!queue.isValid)
                        {
                            logger.Warn($"Could not add Output Queue {queue.queueName} to {name}. Invalid Configuration");
                            continue;
                        }
                        output.Add(queue);
                    }
                    else
                    {
                        logger.Error($"Could not proccess Output Queue {queue.queueName}");
                    }
                } catch (Exception ex) {
                    logger.Error(ex.Message);
                }
            }


            // Create a queue that all the inputs will sent messages to
            try {
                if (!MessageQueue.Exists(inputQueueName))
                {
                    using (MessageQueue t = MessageQueue.Create(inputQueueName)) {
                        this.pipeInputQueue = t;
                        logger.Info($"Created Input Queue {inputQueueName} for pipeline");
                    }
                }
                else
                {
                    this.pipeInputQueue = new MessageQueue(inputQueueName);
                }
            } catch (Exception ex) {
                logger.Error(ex.Message);
            }

            _contextCache = new MemoryCache(name);

            try {
                contextCacheKeyXPath = pipeConfig.Attribute("contextCacheKeyXPath").Value;
            } catch (Exception) {
                contextCacheKeyXPath = null;
            }

            try {
                this.contextCacheExpiry = double.Parse(pipeConfig.Attribute("contextCacheExpiry").Value);
            } catch (Exception) {
                this.contextCacheExpiry = 10.0;
            }

            try {
                this.discardInCache = bool.Parse(pipeConfig.Attribute("discardInCache").Value);
            } catch (Exception) {
                this.discardInCache = false;
            }

            try {
                this.firstOnly = bool.Parse(pipeConfig.Attribute("firstOnly").Value);
            } catch (Exception) {
                this.firstOnly = false;
            }

            try {
                this.useMessageAsKey = bool.Parse(pipeConfig.Attribute("useMessageAsKey").Value);
            } catch (Exception) {
                this.useMessageAsKey = false;
            }



            //Output and reset context
            if (contextCacheKeyXPath != null)
            {
                int contextStatsInterval;
                try {
                    contextStatsInterval = int.Parse(pipeConfig.Attribute("contextStatsInterval").Value);
                } catch (Exception) {
                    contextStatsInterval = 30000;
                }

                resetTimer = new System.Timers.Timer()
                {
                    AutoReset = true,
                    Interval  = contextStatsInterval
                };
                logger.Info($"Context Cache Stats Interval set to {contextStatsInterval}");
                resetTimer.Elapsed += (source, eventArgs) =>
                {
                    statLogger.Info($"Total received: {this.totalReceived}, Total sent: {this.totalSent}");
                    statLogger.Info($">>>> Context Cache Stats for Previous {resetTimer.Interval}ms");

                    try {
                        foreach (var v in statDict.Values)
                        {
                            statLogger.Info(v);
                        }
                    } catch (Exception ex) {
                        logger.Error($"Stats dictionary problem {ex.Message}");
                    } finally {
                        statDict.Clear();
                    }

                    statLogger.Info($"<<<<< End of Context Cache Stats");
                };
                resetTimer.Start();
            }


            /* Only allow if there are output queues configured
             * It is OK if there are no inputs defined. An output only queue may be
             * defined just to specifiy a maxMessages parameter on the queue for size maintenance
             */
            if (output.Count() > 0)
            {
                OK_TO_RUN = true;
            }
            else
            {
                OK_TO_RUN = false;
                logger.Warn($"No inputs or outputs defined for Pipeline {name}");
            }
        }
        public QueueAbstract(XElement defn, IProgress <QueueMonitorMessage> monitorMessageProgress)
        {
            this.monitorMessageProgress = monitorMessageProgress;
            this.definition             = defn;

            try {
                var stylesheet = definition.Attribute("stylesheet").Value;
                this.styleSheets = stylesheet.Split(',').ToList <string>();
                this.bTransform  = true;
            }
            catch (Exception) {
                this.styleSheets = new List <string>();
                this.bTransform  = false;
            }

            try {
                this.queueName = definition.Attribute("queue").Value;
            }
            catch (Exception) {
                this.queueName = "undefined";
            }

            try {
                this.id = definition.Attribute("id").Value;
            }
            catch (Exception) {
                this.id = Guid.NewGuid().ToString();
            }

            try {
                this.name = definition.Attribute("name").Value;
            }
            catch (Exception) {
                this.name = this.queueName;
            }

            try {
                this.createQueue = bool.Parse(definition.Attribute("createQueue").Value);
            }
            catch (Exception) {
                this.createQueue = false;
            }

            try {
                this.xslVersion = definition.Attribute("xslVersion").Value;
            }
            catch (Exception) {
                this.xslVersion = "1.0";
            }

            try {
                this.maxRetry = Convert.ToInt32(definition.Attribute("maxRetry").Value);
            }
            catch (Exception) {
                this.maxRetry = 10;
            }
            try {
                this.priority = Convert.ToInt32(definition.Attribute("priority").Value);

                switch (priority)
                {
                case 0:
                    mqPriority = MessagePriority.Lowest;
                    break;

                case 1:
                    mqPriority = MessagePriority.VeryLow;
                    break;

                case 2:
                    mqPriority = MessagePriority.Low;
                    break;

                case 3:
                    mqPriority = MessagePriority.Normal;
                    break;

                case 4:
                    mqPriority = MessagePriority.AboveNormal;
                    break;

                case 5:
                    mqPriority = MessagePriority.High;
                    break;

                case 6:
                    mqPriority = MessagePriority.VeryHigh;
                    break;

                case 7:
                    mqPriority = MessagePriority.Highest;
                    break;
                }
            }
            catch (Exception) {
                this.priority = 2;
                mqPriority    = MessagePriority.Normal;
            }

            try {
                undeliverableQueue = defn.Attribute("undeliverableQueue").Value;
                CreateQueue(undeliverableQueue);
            }
            catch (Exception) {
                undeliverableQueue = null;
            }

            try {
                bufferQueueName = definition.Attribute("bufferQueueName").Value;
                CreateQueue(bufferQueueName, true);
            }
            catch (Exception) {
                bufferQueueName = null;
            }


            try {
                xpathDestination = definition.Attribute("xpathDestination").Value;
            }
            catch (Exception) {
                xpathDestination = null;
            }
            try {
                xpathContentDestination = definition.Attribute("xpathContentDestination").Value;
            }
            catch (Exception) {
                xpathContentDestination = null;
            }

            // Add any filter expresion
            XElement filtersDefn = defn.Element("filter");

            if (filtersDefn != null)
            {
                // Add the alternate queue to send to if the expression fails (optional)
                try {
                    altQueue = fact.GetQueue(filtersDefn.Element("altqueue"), this.monitorMessageProgress);
                }
                catch (Exception) {
                    altQueue = null;
                }

                /*
                 * At the top level, there is only one Expresssion, which it self can be a compound
                 * expression or a single data filter
                 *
                 * When the Expression itself is constucted, it recurssive creates all the Expression o
                 * filters configured under it
                 */

                // Cycle through the expressions types (and, or, not, xor) to see if any exist
                foreach (string eType in Expression.expressionTypes)
                {
                    XElement exprDefn = filtersDefn.Element(eType);
                    if (exprDefn != null)
                    {
                        expression = new Expression(exprDefn, this);
                    }
                }

                // Cycle through the data filter types (and, or, not, xor) to see if any exist
                foreach (string fType in Expression.filterTypes)
                {
                    XElement filtDefn = filtersDefn.Element(fType);
                    if (filtDefn != null)
                    {
                        topLevelFilter = fact.GetFilter(filtDefn, this);
                    }
                }
            }
            // Call the instance specific setup
            this.isValid = this.SetUp();
        }