示例#1
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            await Task.Run(() => { });

            try {
                TcpClient client = new TcpClient(tcpServerIP, tcpServerPort);

                // Translate the passed message into ASCII and store it as a Byte array.
                Byte[] data = System.Text.Encoding.UTF8.GetBytes(message.payload);

                // Get a client stream for reading and writing.

                NetworkStream stream = client.GetStream();

                // Send the message to the connected TcpServer.
                stream.Write(data, 0, data.Length);

                // Close everything.
                stream.Close();
                client.Close();
            }
            catch (ArgumentNullException e) {
                logger.Error($"ArgumentNullException: {e.Message}");
                message.sent = false;
                return(message);
            }
            catch (SocketException e) {
                logger.Error($"SocketException: {e.Message}");
                message.sent = false;
                return(message);
            }

            message.sent = true;
            return(message);
        }
示例#2
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            try
            {
                bool status = await SendHTTPStuff(mess);

                if (status)
                {
                    logger.Trace($"Sent to {name}");
                }
                else
                {
                    logger.Error($"Failed Sent to {name}");
                    SendToUndeliverableQueue(mess);
                    mess.sent = false;
                    return(mess);
                }
            }
            catch (Exception ex)
            {
                logger.Error(ex, $"Failed Sent to {name}");
                this.SendToUndeliverableQueue(mess);
                mess.sent = false;
                return(mess);
            }
            mess.sent = true;
            return(mess);
        }
示例#3
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            await Task.Run(() => { });

            if (maxMessages != -1)
            {
                MaintainQueueInline();
            }

            //// Connection to the queue
            if (ResetWriteConnect() == maxRetry)
            {
                logger.Error($"Cannot Write to Queue: {queueName}");
                this.SendToUndeliverableQueue(mess);
                mess.sent   = false;
                mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
                return(mess);
            }

            // Locking on send is neccessary because writing to a MSMQ queue is not thread safe
            // and this method may be concurrently called.
            // The lock only protects against write clasehes by this onbject - other objects may
            // indepentantly try to write to the queue.

            lock (sendLock)
            {
                try
                {
                    using (msgQueue)
                    {
                        string payload = mess.payload;

                        try
                        {
                            Message myMessage = new Message(Encoding.ASCII.GetBytes(payload), new ActiveXMessageFormatter());
                            msgQueue.Send(myMessage);
                            mess.sent   = true;
                            mess.status = $"Sent to {queueName}";
                            return(mess);
                        }
                        catch (Exception ex)
                        {
                            logger.Error(ex.Message);
                            logger.Error(ex.StackTrace);
                            mess.sent   = false;
                            mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
                            return(mess);
                        }
                    }
                }
                catch (Exception ex)
                {
                    logger.Error(ex, $"MSMQ Error Sending to {queueName}");
                    SendToUndeliverableQueue(mess);
                    mess.sent   = false;
                    mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
                    return(mess);
                }
            }
        }
示例#4
0
        public void SendToPipe(string message)
        {
            //Check to see if it passes filters and transformations
            ExchangeMessage xm = new ExchangeMessage(message);

            xm = PreAndPostProcess(xm);
            if (xm == null || xm.payload == null)
            {
                logger.Info($"Message did not pass transformation");
                return;
            }
            if (!xm.pass)
            {
                logger.Info($"Message did not pass filter");
                return;
            }
            try {
                Message myMessage = new Message(message, new ActiveXMessageFormatter())
                {
                    Priority = mqPriority
                };
                MessageQueue pipeInputQueue = new MessageQueue(pipeInputQueueName);
                pipeInputQueue.Send(myMessage);
            }
            catch (Exception ex) {
                logger.Error(ex.Message);
                logger.Error(ex.StackTrace);
            }
        }
示例#5
0
        public async Task <ExchangeMessage> Send(ExchangeMessage xm)
        {
            //Do any filtering or transformation first.
            try {
                xm = PreAndPostProcess(xm);
            }
            catch (Exception ex) {
                Console.WriteLine(ex.Message);
                return(null);
            }

            // Check the result of the filtering and transformations
            if (!xm.pass || xm.payload == null)
            {
                xm.status = $"Message blocked by filter to {queueName}";
                xm.pass   = false;
                QXLog(xm?.uuid, "Output Node", "Messages did not pass filter", "PROGRESS");
                return(xm);
            }
            else
            {
                QXLog(xm?.uuid, "Output Node", "Sending Message", "PROGRESS");
                // Send it to the destination.
                // The state of the message should be updated by the particuar output
                xm = await SendToOutputAsync(xm);

                return(xm);
            }
        }
示例#6
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            // Send the message to the bufferQueue, so it is available when a HTTP GET request comes in
            mess = await serviceQueue.SendToOutputAsync(mess);

            mess.status += "Added to HTTPGET Queue";
            return(mess);
        }
示例#7
0
        public async Task <ExchangeMessage> SendMqStuff(ExchangeMessage xm)
        {
            await Task.Run(() => { });

            string messageXML = xm.payload;
            bool   sent       = false;
            int    tries      = 1;

            lock (sendLock)
            {
                do
                {
                    if (tries > maxRetry && maxRetry > 0)
                    {
                        xm.sent   = false;
                        xm.status = $"Exceeded Connection retries on MQ {queueName}";
                        return(xm);
                    }

                    try
                    {
                        using (MQQueueManager queueManager = new MQQueueManager(qMgr, connectionParams))
                        {
                            var     openOptions = MQC.MQOO_OUTPUT + MQC.MQOO_FAIL_IF_QUIESCING;
                            MQQueue queue       = queueManager.AccessQueue(queueName, openOptions);
                            var     message     = new MQMessage
                            {
                                CharacterSet = 1208 // UTF-8
                            };
                            message.WriteString(messageXML);
                            message.Format = MQC.MQFMT_STRING;
                            MQPutMessageOptions putOptions = new MQPutMessageOptions();
                            queue.Put(message, putOptions);
                            queue.Close();
                            sent = true;
                            logger.Trace($"===Message Sent {xm.uuid} to {queueName}");
                        }
                    }
                    catch (Exception ex)
                    {
                        if (!isLogger)
                        {
                            logger.Info("Unable to send message to : " + queueName + " : " + ex.Message);
                        }
                        tries++;
                    }
                    if (!sent)
                    {
                        logger.Trace($"===Message NOT Sent {xm.uuid} to  {queueName}");
                    }
                } while (!sent);
            }
            xm.sent   = true;
            xm.status = $"Sent to  MQ {queueName}";

            return(xm);
        }
示例#8
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            await Task.Run(() =>
            {
                logger.Info($"Sunk Message");
                message.sent = true;
            });

            return(null);
        }
示例#9
0
        public async Task <bool> SendHTTPStuff(ExchangeMessage message)
        {
            string messageXML = message.payload;

            bool sent  = false;
            int  tries = 1;

            do
            {
                if (tries > maxRetry && maxRetry > 0)
                {
                    return(sent);
                }

                try
                {
                    using (var client = new HttpClient())
                    {
                        HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, postURL)
                        {
                            Content = new StringContent(messageXML, Encoding.UTF8, "application/xml")
                        };

                        if (headers != null)
                        {
                            foreach (var item in headers)
                            {
                                requestMessage.Headers.Add(item.Key, item.Value);
                            }
                        }

                        using (HttpResponseMessage response = await client.SendAsync(requestMessage))
                        {
                            if (response.StatusCode == HttpStatusCode.OK)
                            {
                                sent = true;
                                return(sent);
                            }
                            else
                            {
                                throw new Exception();
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    logger.Info("Unable to send message to : " + queueName + " : " + ex.Message);
                    tries++;
                }
            } while (!sent && OK_TO_RUN);

            return(sent);
        }
示例#10
0
        public async Task StartPipeLine()
        {
            // At the pipeline level, the pipe can be configured to use or bypass the
            // filtering and transformation on each of the queues.

            // Pipeline processing is simple, wait for something from the input queue and
            // if is not null, then distribute to the output according to the
            // selected distribution pattern and then repeat

            // The distribution to each of the output queues is done in a seperate async
            // Task so that they do not interfere with each other.

            if (maxMsgPerMinuteProfile != null)
            {
                PrepareProfileThreads();
            }

            foreach (QueueAbstract inQ in input)
            {
                logger.Info($"Starting Async Listener for {inQ.name}");
                var t = Task.Run(() => inQ.StartListener());
            }

            logger.Info($"Pipe {name} running. Input Queues = {input.Count()}, Output Queues = {output.Count()}");
            logger.Info($"Throttle interval = {throttleInterval}");

            while (OK_TO_RUN)
            {
                try {
                    //Get the message from the input
                    ExchangeMessage xm = GetMessage();
                    QXLog("Recieved Message", null, "PROGRESS");

                    if (throttleInterval > 0 || contextAware == true)
                    {
                        QXLog("Sending Message to Context Proceesor", null, "PROGRESS");
                        await ContextProcessorAsync(xm);
                    }
                    else
                    {
                        QXLog("Sending Message to Injector", null, "PROGRESS");
                        await InjectMessage(xm);

                        Thread.Sleep(throttleInterval);
                    }
                } catch (Exception ex) {
                    Console.WriteLine(ex.Message);
                }
            }
            logger.Info($"Stopping Pipeline {name}");
        }
示例#11
0
        private string GetContextKey(ExchangeMessage xm)
        {
            string contextKeyValue = null;

            if (useMessageAsKey)
            {
                QXLog("Context Proceesor", "Entire Message Being Used As Key", "PROGRESS");
                using (SHA256 mySHA256 = SHA256.Create()) {
                    byte[] byteArray = Encoding.UTF8.GetBytes(xm.payload);


                    using (MemoryStream stream = new MemoryStream(byteArray)) {
                        byte[] hashValue = mySHA256.ComputeHash(stream);
                        contextKeyValue = BitConverter.ToString(hashValue);
                    }
                    return(contextKeyValue);
                }
            }
            else if (contextCacheKeyXPath == "*")
            {
                return("ALL_MESSAGES");
            }
            else
            {
                // Look at the XML to get the contextKey
                XmlDocument doc = new XmlDocument();
                doc.LoadXml(xm.payload);
                XmlNamespaceManager ns = new XmlNamespaceManager(doc.NameTable);

                foreach (KeyValuePair <string, string> item in Exchange.nsDict)
                {
                    ns.AddNamespace(item.Key, item.Value);
                }


                try {
                    XmlNode node = doc.SelectSingleNode(contextCacheKeyXPath, ns);
                    contextKeyValue = node.InnerText;
                    logger.Info($"Context Key for Message = {contextKeyValue}");
                } catch {
                    return(null);
                }
                QXLog("Context Proceesor", $"Context Key Retrieves Context Key = {contextKeyValue}", "PROGRESS");

                return(contextKeyValue);
            }
        }
示例#12
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            var config = new ProducerConfig {
                BootstrapServers = this.bootStrapServers
            };

            // Set the topic bassed on the content of the message if configured
            mess = SetDestinationFromMessage(mess);
            if (mess.destinationSet)
            {
                topic = queueName;
            }
            else
            {
                if (topic == null)
                {
                    mess.sent = false;
                    return(mess);
                }
            }

            using (var p = new ProducerBuilder <string, string>(config).Build())
            {
                try
                {
                    var dr = await p.ProduceAsync(topic, new Message <string, string> {
                        Key = key, Value = mess.payload
                    });
                }
                catch (Exception e)
                {
                    logger.Error(e.Message);
                    logger.Error(e);
                    logger.Error($"Unable to deliver to Kafka Server on topic {topic}");
                    SendToUndeliverableQueue(mess);
                    mess.sent   = false;
                    mess.status = $"Unable to Send to Kafka Topic = {topic}";
                    return(mess);
                }
                logger.Info($"Kafka sent message to {topic}");
            }

            mess.sent   = true;
            mess.status = $"Sent to Kafka Topic = {topic}";
            return(mess);
        }
示例#13
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            int    count    = 0;
            string countStr = count.ToString();

            while (countStr.Length < 6)
            {
                countStr = "0" + countStr;
            }


            string fileNameOnly = Path.GetFileNameWithoutExtension(fullPath);
            string extension    = Path.GetExtension(fullPath);
            string path         = Path.GetDirectoryName(fullPath);
            string newFullPath;
            string tempFileName = string.Format("{0}{1}", fileNameOnly, countStr);

            newFullPath = Path.Combine(path, tempFileName + extension);

            while (File.Exists(newFullPath))
            {
                count++;
                countStr = count.ToString();

                while (countStr.Length < 6)
                {
                    countStr = "0" + countStr;
                }
                tempFileName = string.Format("{0}{1}", fileNameOnly, countStr);
                newFullPath  = Path.Combine(path, tempFileName + extension);
            }

            try {
                await Task.Run(() => File.WriteAllText(newFullPath, mess.payload));

                mess.sent   = true;
                mess.status = $"Sent file to {newFullPath}";
            }
            catch (Exception) {
                mess.sent   = false;
                mess.status = $"Unable to write file to {newFullPath}";
            }

            return(mess);
        }
示例#14
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            await Task.Run(() =>
            {
                {
                    if (!OK_TO_SEND)
                    {
                        logger.Warn("FTP Interface not configured correctly");
                        return;
                    }

                    string guid = Guid.NewGuid().ToString();

                    if (!ftpURL.EndsWith("/"))
                    {
                        ftpURL = ftpURL + "/";
                    }

                    if (!directory.StartsWith("/"))
                    {
                        directory = "/" + directory;
                    }
                    if (!directory.EndsWith("/"))
                    {
                        directory = directory + "/";
                    }
                    using (var ftpClient = new FtpClient(new Uri(ftpURL))) {
                        try {
                            if (useFTPS)
                            {
                                ftpClient.EncryptionMode         = FtpEncryptionMode.Explicit;
                                ftpClient.ValidateAnyCertificate = true;
                                ftpClient.Connect();
                            }

                            ftpClient.Upload(Encoding.UTF8.GetBytes(message.payload), directory + guid);
                        } catch (Exception ex) {
                            logger.Error(ex.Message);
                        }
                    }
                }
            });

            return(null);
        }
示例#15
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            // Create a new MQTT client.

            var msg = new MqttApplicationMessageBuilder()
                      .WithTopic(topic)
                      .WithPayload(message.payload)
                      .WithExactlyOnceQoS()
                      .WithRetainFlag()
                      .Build();

            //"broker.hivemq.com", 1883)
            if (serverType == "tcp")
            {
                options = new MqttClientOptionsBuilder().WithTcpServer(mqttServer, mqttServerPort).Build();
            }

            //"broker.hivemq.com:8000/mqtt"
            if (serverType == "ws")
            {
                options = new MqttClientOptionsBuilder().WithWebSocketServer(mqttServerURL).Build();
            }
            if (!mqttClient.IsConnected)
            {
                mqttClient.ConnectAsync(options).Wait();
            }

            MqttClientPublishResult result = await mqttClient.PublishAsync(msg, CancellationToken.None);

            if (result.ReasonCode == MqttClientPublishReasonCode.Success)
            {
                message.sent   = true;
                message.status = $"Sent to MQTT topic {queueName}";
                return(message);
            }
            else
            {
                message.sent   = true;
                message.status = $"MQTT Send Failure {result.ReasonString}";
                logger.Error($"MQTT Send Failure {result.ReasonString}");
                return(message);
            }
        }
示例#16
0
        public void SendToUndeliverableQueue(ExchangeMessage xm)
        {
            // If messages cant be delivered, they can be sent to a local undeliverable queue
            // defined on a per queue basis. No reason why the one undeliverable queue cant be used
            // for all of the queue.


            if (isLogger)
            {
                // If this is a logger, failure messages won't be sent to the undeliverable queue
                logger.Error("Log Message could not be sent to logger");
                return;
            }

            QXLog(xm?.uuid, "Output Node Send Failure", "Message Could Not Be Delivered to the Output Node", "PROGRESS");

            if (undeliverableQueue == null)
            {
                logger.Debug($"No Undeliverable Message Queue has been defined for {queueName}");
                QXLog(xm?.uuid, "Output Node Send Failure", "No Undeliverable Queue Defined", "WARNING");

                return;
            }

            lock (undevlLock) {
                try {
                    using (MessageQueue er = new MessageQueue(undeliverableQueue)) {
                        QXLog(xm?.uuid, "Output Node Send Failure", "Sending Message to Undeliverable Queue", "PROGRESS");
                        er.Send(xm);
                        QXLog(xm?.uuid, "Output Node Send Failure", "Message Sent to Undeliverable Queue", "PROGRESS");


                        logger.Info("Message Sent to Undeliverble Queue");
                    }
                }
                catch (Exception ex) {
                    QXLog(xm?.uuid, "Output Node Send Failure", "Message Could Not Be Sent To Undeliverable Queue", "ERROR");

                    logger.Error("Unable to Send to Undeliverble Queue");
                    logger.Error(ex.StackTrace);
                }
            }
        }
示例#17
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
        {
            if (maxMessages != -1)
            {
                MaintainQueueInline();
            }

            try
            {
                mess = await SendMqStuff(mess);

                if (mess.sent)
                {
                    logger.Debug($"MQ Sent to {name}");
                    mess.sent   = true;
                    mess.status = $"Sent to {queueName}";
                    return(mess);
                }
                else
                {
                    logger.Error($"MQ Failed Sent to  {name}");
                    SendToUndeliverableQueue(mess);
                    mess.sent   = false;
                    mess.status = $"Unable to Send to {queueName}";
                    return(mess);
                }
            }
            catch (Exception ex)
            {
                if (!isLogger)
                {
                    logger.Error(ex, $"MQ Failed Sent to {name}");
                }
                SendToUndeliverableQueue(mess);
                mess.sent   = false;
                mess.status = $"Unable to Send to {queueName}";
                return(mess);
            }
        }
示例#18
0
        protected ExchangeMessage SetDestinationFromMessage(ExchangeMessage mess)
        {
            if (xpathDestination != null)
            {
                queueName = GetDestinationFromMessage(mess.payload, true);
                if (queueName == null)
                {
                    mess.status         = "Destination could not be found in the XPATH expression";
                    mess.destinationSet = false;
                    return(mess);
                }
                else
                {
                    mess.destinationSet = true;
                    return(mess);
                }
            }

            if (xpathContentDestination != null)
            {
                queueName = GetDestinationFromMessage(mess.payload, false);
                if (queueName == null)
                {
                    mess.status         = "Destination Topic could not be found in the XPATH expression";
                    mess.destinationSet = false;
                    return(mess);
                }
                else
                {
                    mess.destinationSet = true;
                    return(mess);
                }
            }

            mess.destinationSet = false;
            return(mess);
        }
示例#19
0
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            await Task.Run(() =>
            {
                {
                    if (!OK_TO_SEND)
                    {
                        logger.Warn("SMTP Interface not configured correctly");
                        return;
                    }
                    string subject = smtpsubject;

                    try {
                        var mailmessage = new MimeMessage();
                        mailmessage.From.Add(new MailboxAddress(smtpfromUser, smtpfromEmail));
                        mailmessage.To.Add(new MailboxAddress(smtptoUser, smtptoEmail));
                        mailmessage.Subject = subject;

                        if (smtpAttachment)
                        {
                            var body = new TextPart("plain")
                            {
                                Text = "Message is in attachment"
                            };
                            var attachment = new MimePart("text", "plain")
                            {
                                ContentDisposition = new ContentDisposition(ContentDisposition.Attachment),
                                FileName           = Path.GetFileName(smtpattachname)
                            };

                            byte[] byteArray    = Encoding.ASCII.GetBytes(message.payload);
                            MemoryStream stream = new MemoryStream(byteArray);
                            attachment.Content  = new MimeContent(stream, ContentEncoding.Default);

                            var multipart = new Multipart("mixed")
                            {
                                body,
                                attachment
                            };

                            // now set the multipart/mixed as the message body
                            mailmessage.Body = multipart;
                        }
                        else
                        {
                            mailmessage.Body = new TextPart("plain")
                            {
                                Text = message.payload
                            };
                        }


                        using (var client = new SmtpClient()) {
                            // For demo-purposes, accept all SSL certificates (in case the server supports STARTTLS)
                            client.ServerCertificateValidationCallback = (s, c, h, e) => true;

                            client.Connect(smtphost, smtpport, smtpuseSSL);
                            //client.Connect("smtp.mail.yahoo.com", 465, true);


                            // Note: only needed if the SMTP server requires authentication
                            if (smtpuser != null)
                            {
                                client.Authenticate(smtpuser, smtppass);
                            }
                            //                  client.Authenticate("*****@*****.**", "!@aiw2dihsf!@");

                            client.Send(mailmessage);
                            client.Disconnect(true);
                        }
                    }
                    catch (Exception e) {
                        logger.Error($"SMTP Client Send Exception: {e.Message}");
                    }
                }
            });

            return(null);
        }
示例#20
0
        public ExchangeMessage PreAndPostProcess(ExchangeMessage xm)
        {
            QXLog(xm?.uuid, "Output Node: Message Recieved From Pipe", null, "PROGRESS");


            // payload is the actual content of the message to be send
            string message = xm.payload;

            // After the delivery or prior to sending a message, it can be filtered or
            // transformed according to the configuration for each queue

            // The Expression object manages the evaluation of all the filters
            if (expression != null)
            {
                bool pass = expression.Pass(message);

                logger.Info($"Top Expression Evaluated {pass}");
                if (!pass)
                {
                    if (altQueue != null)
                    {
                        logger.Info($"Sending to Alt Queue {altQueue.name}");
                        QXLog(xm?.uuid, "Message did not pass filter", "Sending to Alt Queue", "PROGRESS");
                        Task.Run(async() => { _ = await altQueue.Send(xm); });
                    }
                    else
                    {
                        QXLog(xm?.uuid, "Message did not pass filter", "No Alt Queue Configured", "WARNING");
                    }
                    xm.pass = false;
                    return(xm);
                }
                else
                {
                    xm.pass = true;
                }
            }

            if (topLevelFilter != null)
            {
                bool pass = topLevelFilter.Pass(message);

                logger.Info($"Top Filter Evaluated {pass}");
                if (!pass)
                {
                    if (altQueue != null)
                    {
                        logger.Info($"Sending to Alt Queue {altQueue.name}");
                        QXLog(xm?.uuid, "Message did not pass filter", "Sending to Alt Queue", "PROGRESS");
                        _ = altQueue.Send(xm);
                    }
                    else
                    {
                        QXLog(xm?.uuid, "Message did not pass filter", "No Alt Queue Configured", "WARNING");
                    }
                    xm.pass = false;
                    return(xm);
                }
                else
                {
                    xm.pass = true;
                }
            }


            // If a XSLT transform has been specified
            if (bTransform)
            {
                QXLog(xm?.uuid, "Starting Message Transformation", null, "PROGRESS");
                message = Transform(message, xslVersion);
                QXLog(xm?.uuid, "Message Transformation Complete", null, "PROGRESS");

                xm.transformed = true;
            }
            else
            {
                xm.transformed = false;
            }

            if (message == null || message.Length == 0)
            {
                logger.Info("Message blocked by XSL Transform of Zero Length");
                xm.payload = null;
                xm.status  = "Message blocked by XSL Transform. Null or Zero Length";

                QXLog(xm?.uuid, "Messaage Transformation", "Transformation resulted in a message of zero length", "WARNING");

                return(xm);
            }

            xm.payload = message;
            QXLog(xm?.uuid, "Output Node: Post Processing Complete", null, "PROGRESS");
            return(xm);
        }
示例#21
0
 public abstract Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message);
示例#22
0
        public async Task ContextProcessorAsync(ExchangeMessage xm)
        {
            if (!xm.pass)
            {
                return;
            }

            // If no contextKey has been provided, just inject the message straight away
            if (contextCacheKeyXPath == null && !useMessageAsKey)
            {
                await InjectMessage(xm);

                return;
            }

            string contextKeyValue = GetContextKey(xm);

            if (contextKeyValue == null)
            {
                await InjectMessage(xm);

                return;
            }

            try {
                Queue <ExchangeMessage> bufferMemoryQueue = null;
                System.Timers.Timer     bufferPopperTimer = null;


                if (_bufferMemoryQueueDict.ContainsKey(contextKeyValue) && _bufferTimerDict.ContainsKey(contextKeyValue))
                {
                    bufferMemoryQueue = _bufferMemoryQueueDict[contextKeyValue];
                    bufferPopperTimer = _bufferTimerDict[contextKeyValue];
                }
                else
                {
                    // They don't exist, so they need to be created.
                    logger.Trace($"Creating queue and timer for {contextKeyValue}");

                    //Create the queue
                    bufferMemoryQueue = new Queue <ExchangeMessage>();
                    _bufferMemoryQueueDict.Add(contextKeyValue, bufferMemoryQueue);

                    //Create the timer
                    bufferPopperTimer = CreatePopperTask(xm, bufferMemoryQueue, contextKeyValue);
                    _bufferTimerDict.Add(contextKeyValue, bufferPopperTimer);
                }

                ContextStats stats = null;
                if (statDict.ContainsKey(contextKeyValue))
                {
                    stats = statDict[contextKeyValue];
                }
                else
                {
                    ContextStats v = new ContextStats {
                        key = contextKeyValue
                    };
                    statDict.Add(contextKeyValue, v);
                    stats = v;
                }

                stats.recieved++;

                this.totalReceived++;

                if (_contextCache.Contains(contextKeyValue) && this.discardInCache)
                {
                    logger.Info($"Message found in Cache - Discarding. Message Hash {contextKeyValue}");
                    return;
                }

                if (mostRecentOnly)
                {
                    bufferMemoryQueue.Clear();
                }

                bufferMemoryQueue.Enqueue(xm);

                // So the message is enqueued, now just work out how long how to set the popper

                //Let's deal with it if it is not a firstOnly

                if (!firstOnly)
                {
                    if (!_contextCache.Contains(contextKeyValue))
                    {
                        //It's not in the cachce

                        if (!bufferPopperTimer.Enabled)
                        {
                            //If the popper isn't already enabled, enable it to pop straight away (5ms)
                            bufferPopperTimer.Interval = 5.0;
                            bufferPopperTimer.Enabled  = true;
                            bufferPopperTimer.Start();
                        }
                    }
                    else
                    {
                        //It's in tha cache,
                        if (!bufferPopperTimer.Enabled)
                        {
                            //If the popper isn't already enabled, then set the popper. (If it is in the cache, then really, the popper should already be set)
                            bufferPopperTimer.Interval = contextCacheExpiry * 1000;
                            bufferPopperTimer.Enabled  = true;
                            bufferPopperTimer.Start();
                        }
                    }
                    return;
                }

                if (firstOnly)
                {
                    //It's not in the cache

                    bool inCache   = _contextCache.Contains(contextKeyValue);
                    bool firstDone = _firstProcessedCompleted.Contains(contextKeyValue);
                    if (!inCache || firstDone)
                    {
                        _contextCache.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));
                        if (!bufferPopperTimer.Enabled)
                        {
                            bufferPopperTimer.Interval = 10.0;
                            bufferPopperTimer.Enabled  = true;
                            bufferPopperTimer.Start();
                        }
                    }
                    else
                    {
                        if (!bufferPopperTimer.Enabled)
                        {
                            bufferPopperTimer.Interval = contextCacheExpiry * 1000;
                            bufferPopperTimer.Enabled  = true;
                            bufferPopperTimer.Start();
                        }
                    }
                    return;
                }
            } catch (Exception ex) {
                logger.Error(ex);
            }
        }
示例#23
0
 public new async void Send(ExchangeMessage xm)
 {
     logger.Debug($"Sending to {xm.uuid} IBMMQ {queueName}");
     await SendToOutputAsync(xm);
 }
示例#24
0
        //public override ExchangeMessage Listen(bool immediateReturn, int priorityWait)
        //{
        //    // The serviceQueue is monitored and messages are returned as the appear on the queue
        //    logger.Debug("Listening to KAFKA QUEUE");
        //    return (serviceQueue.Listen(immediateReturn, priorityWait));
        //}

        public new async void Send(ExchangeMessage xm)
        {
            logger.Debug($"Sending to {xm.uuid} Kafka Topic {topic}");
            await SendToOutputAsync(xm);
        }
        public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
        {
            await Task.Run(() => { });


            // Set the queueName bassed on the content of the message if configured
            message = SetDestinationFromMessage(message);
            if (!message.destinationSet && queueName == null)
            {
                message.sent = false;
                return(message);
            }

            try {
                var factory = new ConnectionFactory()
                {
                    HostName = connection
                };
                factory.UserName    = user;
                factory.Password    = pass;
                factory.VirtualHost = vhost;
                factory.HostName    = connection;
                factory.Port        = port;
                using (var conn = factory.CreateConnection()) {
                    using (var channel = conn.CreateModel()) {
                        try {
                            QueueDeclareOk ch = channel.QueueDeclare(queue: queueName,
                                                                     durable: true,
                                                                     exclusive: false,
                                                                     autoDelete: false,
                                                                     arguments: null);
                        } catch (Exception ex) {
                            logger.Error(ex.Message);
                            logger.Error(ex.StackTrace);
                            message.sent   = false;
                            message.status = $"Error  Sending to Rabbit Topic {queueName}";
                            return(message);
                        }
                        var body = Encoding.UTF8.GetBytes(message.payload);

                        try {
                            channel.BasicPublish(exchange: "",
                                                 routingKey: queueName,
                                                 basicProperties: null,
                                                 body: body);
                        } catch (Exception ex) {
                            logger.Error(ex.Message);
                            logger.Error(ex.StackTrace);
                            SendToUndeliverableQueue(message);
                            message.sent   = false;
                            message.status = $"Error Sending to Rabbit Queue {queueName}";
                            return(message);
                        }
                        message.sent   = true;
                        message.status = $"Sent to Rabbit Queue {queueName}";
                        return(message);
                    }
                }
            } catch (Exception e) {
                logger.Error(e.Message);
                message.sent = false;
                return(message);
            }
        }
示例#26
0
        private System.Timers.Timer CreatePopperTask(ExchangeMessage xm, Queue <ExchangeMessage> bufferMemoryQueue, string contextKeyValue)
        {
            double interval = contextCacheExpiry * 1000 + 100.0;

            logger.Info($"Creating Popper Task. Message Hash: {contextKeyValue}");

            System.Timers.Timer bufferPopperTimer = new System.Timers.Timer {
                Interval = interval,
                Enabled  = false
            };
            bufferPopperTimer.Elapsed += async(source, eventArgs) =>
            {
                bufferPopperTimer.AutoReset = false;
                bufferPopperTimer.Enabled   = false;

                if (_firstProcessedCompleted.Contains(contextKeyValue))
                {
                    //  logger.Info("Pooper Expired - Marking it cleared <<<<<<<<<<<<<<<<<<");
                    _firstProcessedCleared.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));
                }

                if (bufferMemoryQueue.Count != 0)
                {
                    logger.Trace($"Injection Popper Timer Went Off for key = {contextKeyValue} - {bufferMemoryQueue.Count} Messages on queue");

                    _firstProcessedCompleted.AddOrGetExisting(contextKeyValue, contextKeyValue, DateTime.Now.AddHours(18));
                    _contextCache.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));

                    if (bufferMemoryQueue.Count > 0)
                    {
                        await InjectMessage(bufferMemoryQueue.Dequeue());
                    }
                    else
                    {
                        return;
                    }
                    if (firstOnly && _firstProcessedCleared.Contains(contextKeyValue))
                    {
                        bufferPopperTimer.Interval = 10.0;
                    }
                    else
                    {
                        bufferPopperTimer.Interval = contextCacheExpiry * 1000;
                    }
                    bufferPopperTimer.AutoReset = true;
                    bufferPopperTimer.Enabled   = true;

                    try {
                        statDict[contextKeyValue].send++;
                        statLogger.Trace(statDict[contextKeyValue]);
                    } catch (Exception ex) {
                        logger.Error($"Stats dictionary problem {ex.Message}");
                    }
                    this.totalSent++;
                }
                else
                {
                    logger.Trace($"Reinjection Popper Timer Went Off for key = {contextKeyValue} - No Message to Process");
                    if (_firstProcessedCleared.Contains(contextKeyValue))
                    {
                        bufferPopperTimer.Enabled   = false;
                        bufferPopperTimer.AutoReset = false;
                    }
                    else
                    {
                        bufferPopperTimer.Interval  = contextCacheExpiry * 1000;
                        bufferPopperTimer.Enabled   = true;
                        bufferPopperTimer.AutoReset = true;
                    }
                }
            };

            return(bufferPopperTimer);
        }
示例#27
0
        protected async Task InjectMessage(ExchangeMessage xm)
        {
            if (xm != null)
            {
                if (!xm.pass)
                {
                    logger.Trace($"Blocked by incoming filter. Message {xm.uuid }");
                    return;
                }

                msgRecieved++;
                logger.Trace($"Processing message {xm.uuid }  [{msgRecieved}]");
                xm.status = "Message Recieved";

                // Pipeline specific logging
                QXLog("Starting Output Meesage Processing", null, "PROGRESS");

                if (randomDistribution)
                {
                    // Distributes the message to a random output

                    Random rnd   = new Random();
                    int    index = rnd.Next(0, output.Count);

                    logger.Trace($"Random Distributor. Sending message {xm.uuid } to {output[index].name}");
                    QXLog("Output Meesage Processing", "Distributing to Random Output", "PROGRESS");

                    if (outputIsolation)
                    {
                        QXLog("Async Message Start", "Distribution to Random Output", "PROGRESS");
                        _ = Task.Run(() => SendAndLog(output[index], xm));
                        QXLog("Async Message End", "Distribution to Random Output", "PROGRESS");
                    }
                    else
                    {
                        QXLog("Sync Message Start", "Distribution to Random Output", "PROGRESS");
                        _ = await SendAndLog(output[index], xm);

                        QXLog("Sync Message End", "Distribution to Random Output", "PROGRESS");
                    }
                }
                else if (roundRobinDistribution)
                {
                    // Distributes the message only once to each queue in turn
                    int index = nextOutput % output.Count;

                    logger.Trace($"Round Robin Distributor. Sending message {xm.uuid } to {output[index].name}");
                    QXLog("Output Message Processing", "Distribution to Round Robin Output", "PROGRESS");

                    if (outputIsolation)
                    {
                        QXLog("Async Message Start", "Distribution to Round Robin Output", "PROGRESS");
                        _ = Task.Run(() => SendAndLog(output[index], xm));
                        QXLog("Async Message End", "Distribution to Round Robin Output", "PROGRESS");
                    }
                    else
                    {
                        QXLog("Sync Message Start", "Distribution to Round Robin Output", "PROGRESS");
                        _ = await SendAndLog(output[index], xm);

                        QXLog("Sync Message End", "Distribution to Round Robin Output", "PROGRESS");
                    }

                    nextOutput++;
                }
                else
                {
                    // Distribute the message to all the outputs
                    if (outputIsolation)
                    {
                        // Wont wait for the Send to complete before completing
                        foreach (QueueAbstract q in output)
                        {
                            QXLog("Async Message Start", "Normal Distribution", "PROGRESS");
                            _ = Task.Run(() => SendAndLog(q, xm));
                            QXLog("Async Message End", "Normal Distribution", "PROGRESS");
                        }
                    }
                    else
                    {
                        // Will wait for all Sends to complete before proceeding
                        QXLog("Sync Message Start", "Normal Distribution", "PROGRESS");

                        foreach (QueueAbstract q in output)
                        {
                            logger.Info($"-->{name} --> sending message {xm?.uuid } to {q.name}");
                            _ = await SendAndLog(q, xm);

                            //if (xm.sent)
                            //{
                            //    if (q.queueName != null)
                            //    {
                            //        msgCount[q.queueName] = msgCount[q.queueName] + 1;
                            //        logger.Trace($"{q.queueName} = {msgCount[q.queueName]}");
                            //    }
                            //    else
                            //    {
                            //        msgCount[q.name] = msgCount[q.name] + 1;
                            //        logger.Trace($"{q.name} = {msgCount[q.name]}");
                            //    }
                            //}
                        }
                        QXLog("Sync Message End", "Normal Distribution", "PROGRESS");
                    }
                }
            }
        }
示例#28
0
        private async Task <ExchangeMessage> SendAndLog(QueueAbstract queue, ExchangeMessage message)
        {
            message = await queue.Send(message);

            return(message);
        }