public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
await Task.Run(() => { });
try {
TcpClient client = new TcpClient(tcpServerIP, tcpServerPort);
// Translate the passed message into ASCII and store it as a Byte array.
Byte[] data = System.Text.Encoding.UTF8.GetBytes(message.payload);
// Get a client stream for reading and writing.
NetworkStream stream = client.GetStream();
// Send the message to the connected TcpServer.
stream.Write(data, 0, data.Length);
// Close everything.
stream.Close();
client.Close();
}
catch (ArgumentNullException e) {
logger.Error($"ArgumentNullException: {e.Message}");
message.sent = false;
return(message);
}
catch (SocketException e) {
logger.Error($"SocketException: {e.Message}");
message.sent = false;
return(message);
}
message.sent = true;
return(message);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
try
{
bool status = await SendHTTPStuff(mess);
if (status)
{
logger.Trace($"Sent to {name}");
}
else
{
logger.Error($"Failed Sent to {name}");
SendToUndeliverableQueue(mess);
mess.sent = false;
return(mess);
}
}
catch (Exception ex)
{
logger.Error(ex, $"Failed Sent to {name}");
this.SendToUndeliverableQueue(mess);
mess.sent = false;
return(mess);
}
mess.sent = true;
return(mess);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
await Task.Run(() => { });
if (maxMessages != -1)
{
MaintainQueueInline();
}
//// Connection to the queue
if (ResetWriteConnect() == maxRetry)
{
logger.Error($"Cannot Write to Queue: {queueName}");
this.SendToUndeliverableQueue(mess);
mess.sent = false;
mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
return(mess);
}
// Locking on send is neccessary because writing to a MSMQ queue is not thread safe
// and this method may be concurrently called.
// The lock only protects against write clasehes by this onbject - other objects may
// indepentantly try to write to the queue.
lock (sendLock)
{
try
{
using (msgQueue)
{
string payload = mess.payload;
try
{
Message myMessage = new Message(Encoding.ASCII.GetBytes(payload), new ActiveXMessageFormatter());
msgQueue.Send(myMessage);
mess.sent = true;
mess.status = $"Sent to {queueName}";
return(mess);
}
catch (Exception ex)
{
logger.Error(ex.Message);
logger.Error(ex.StackTrace);
mess.sent = false;
mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
return(mess);
}
}
}
catch (Exception ex)
{
logger.Error(ex, $"MSMQ Error Sending to {queueName}");
SendToUndeliverableQueue(mess);
mess.sent = false;
mess.status = $"Unable to deliver to {queueName}. Sent to Undeliverable";
return(mess);
}
}
}
public void SendToPipe(string message)
{
//Check to see if it passes filters and transformations
ExchangeMessage xm = new ExchangeMessage(message);
xm = PreAndPostProcess(xm);
if (xm == null || xm.payload == null)
{
logger.Info($"Message did not pass transformation");
return;
}
if (!xm.pass)
{
logger.Info($"Message did not pass filter");
return;
}
try {
Message myMessage = new Message(message, new ActiveXMessageFormatter())
{
Priority = mqPriority
};
MessageQueue pipeInputQueue = new MessageQueue(pipeInputQueueName);
pipeInputQueue.Send(myMessage);
}
catch (Exception ex) {
logger.Error(ex.Message);
logger.Error(ex.StackTrace);
}
}
public async Task <ExchangeMessage> Send(ExchangeMessage xm)
{
//Do any filtering or transformation first.
try {
xm = PreAndPostProcess(xm);
}
catch (Exception ex) {
Console.WriteLine(ex.Message);
return(null);
}
// Check the result of the filtering and transformations
if (!xm.pass || xm.payload == null)
{
xm.status = $"Message blocked by filter to {queueName}";
xm.pass = false;
QXLog(xm?.uuid, "Output Node", "Messages did not pass filter", "PROGRESS");
return(xm);
}
else
{
QXLog(xm?.uuid, "Output Node", "Sending Message", "PROGRESS");
// Send it to the destination.
// The state of the message should be updated by the particuar output
xm = await SendToOutputAsync(xm);
return(xm);
}
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
// Send the message to the bufferQueue, so it is available when a HTTP GET request comes in
mess = await serviceQueue.SendToOutputAsync(mess);
mess.status += "Added to HTTPGET Queue";
return(mess);
}
public async Task <ExchangeMessage> SendMqStuff(ExchangeMessage xm)
{
await Task.Run(() => { });
string messageXML = xm.payload;
bool sent = false;
int tries = 1;
lock (sendLock)
{
do
{
if (tries > maxRetry && maxRetry > 0)
{
xm.sent = false;
xm.status = $"Exceeded Connection retries on MQ {queueName}";
return(xm);
}
try
{
using (MQQueueManager queueManager = new MQQueueManager(qMgr, connectionParams))
{
var openOptions = MQC.MQOO_OUTPUT + MQC.MQOO_FAIL_IF_QUIESCING;
MQQueue queue = queueManager.AccessQueue(queueName, openOptions);
var message = new MQMessage
{
CharacterSet = 1208 // UTF-8
};
message.WriteString(messageXML);
message.Format = MQC.MQFMT_STRING;
MQPutMessageOptions putOptions = new MQPutMessageOptions();
queue.Put(message, putOptions);
queue.Close();
sent = true;
logger.Trace($"===Message Sent {xm.uuid} to {queueName}");
}
}
catch (Exception ex)
{
if (!isLogger)
{
logger.Info("Unable to send message to : " + queueName + " : " + ex.Message);
}
tries++;
}
if (!sent)
{
logger.Trace($"===Message NOT Sent {xm.uuid} to {queueName}");
}
} while (!sent);
}
xm.sent = true;
xm.status = $"Sent to MQ {queueName}";
return(xm);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
await Task.Run(() =>
{
logger.Info($"Sunk Message");
message.sent = true;
});
return(null);
}
public async Task <bool> SendHTTPStuff(ExchangeMessage message)
{
string messageXML = message.payload;
bool sent = false;
int tries = 1;
do
{
if (tries > maxRetry && maxRetry > 0)
{
return(sent);
}
try
{
using (var client = new HttpClient())
{
HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, postURL)
{
Content = new StringContent(messageXML, Encoding.UTF8, "application/xml")
};
if (headers != null)
{
foreach (var item in headers)
{
requestMessage.Headers.Add(item.Key, item.Value);
}
}
using (HttpResponseMessage response = await client.SendAsync(requestMessage))
{
if (response.StatusCode == HttpStatusCode.OK)
{
sent = true;
return(sent);
}
else
{
throw new Exception();
}
}
}
}
catch (Exception ex)
{
logger.Info("Unable to send message to : " + queueName + " : " + ex.Message);
tries++;
}
} while (!sent && OK_TO_RUN);
return(sent);
}
public async Task StartPipeLine()
{
// At the pipeline level, the pipe can be configured to use or bypass the
// filtering and transformation on each of the queues.
// Pipeline processing is simple, wait for something from the input queue and
// if is not null, then distribute to the output according to the
// selected distribution pattern and then repeat
// The distribution to each of the output queues is done in a seperate async
// Task so that they do not interfere with each other.
if (maxMsgPerMinuteProfile != null)
{
PrepareProfileThreads();
}
foreach (QueueAbstract inQ in input)
{
logger.Info($"Starting Async Listener for {inQ.name}");
var t = Task.Run(() => inQ.StartListener());
}
logger.Info($"Pipe {name} running. Input Queues = {input.Count()}, Output Queues = {output.Count()}");
logger.Info($"Throttle interval = {throttleInterval}");
while (OK_TO_RUN)
{
try {
//Get the message from the input
ExchangeMessage xm = GetMessage();
QXLog("Recieved Message", null, "PROGRESS");
if (throttleInterval > 0 || contextAware == true)
{
QXLog("Sending Message to Context Proceesor", null, "PROGRESS");
await ContextProcessorAsync(xm);
}
else
{
QXLog("Sending Message to Injector", null, "PROGRESS");
await InjectMessage(xm);
Thread.Sleep(throttleInterval);
}
} catch (Exception ex) {
Console.WriteLine(ex.Message);
}
}
logger.Info($"Stopping Pipeline {name}");
}
private string GetContextKey(ExchangeMessage xm)
{
string contextKeyValue = null;
if (useMessageAsKey)
{
QXLog("Context Proceesor", "Entire Message Being Used As Key", "PROGRESS");
using (SHA256 mySHA256 = SHA256.Create()) {
byte[] byteArray = Encoding.UTF8.GetBytes(xm.payload);
using (MemoryStream stream = new MemoryStream(byteArray)) {
byte[] hashValue = mySHA256.ComputeHash(stream);
contextKeyValue = BitConverter.ToString(hashValue);
}
return(contextKeyValue);
}
}
else if (contextCacheKeyXPath == "*")
{
return("ALL_MESSAGES");
}
else
{
// Look at the XML to get the contextKey
XmlDocument doc = new XmlDocument();
doc.LoadXml(xm.payload);
XmlNamespaceManager ns = new XmlNamespaceManager(doc.NameTable);
foreach (KeyValuePair <string, string> item in Exchange.nsDict)
{
ns.AddNamespace(item.Key, item.Value);
}
try {
XmlNode node = doc.SelectSingleNode(contextCacheKeyXPath, ns);
contextKeyValue = node.InnerText;
logger.Info($"Context Key for Message = {contextKeyValue}");
} catch {
return(null);
}
QXLog("Context Proceesor", $"Context Key Retrieves Context Key = {contextKeyValue}", "PROGRESS");
return(contextKeyValue);
}
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
var config = new ProducerConfig {
BootstrapServers = this.bootStrapServers
};
// Set the topic bassed on the content of the message if configured
mess = SetDestinationFromMessage(mess);
if (mess.destinationSet)
{
topic = queueName;
}
else
{
if (topic == null)
{
mess.sent = false;
return(mess);
}
}
using (var p = new ProducerBuilder <string, string>(config).Build())
{
try
{
var dr = await p.ProduceAsync(topic, new Message <string, string> {
Key = key, Value = mess.payload
});
}
catch (Exception e)
{
logger.Error(e.Message);
logger.Error(e);
logger.Error($"Unable to deliver to Kafka Server on topic {topic}");
SendToUndeliverableQueue(mess);
mess.sent = false;
mess.status = $"Unable to Send to Kafka Topic = {topic}";
return(mess);
}
logger.Info($"Kafka sent message to {topic}");
}
mess.sent = true;
mess.status = $"Sent to Kafka Topic = {topic}";
return(mess);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
int count = 0;
string countStr = count.ToString();
while (countStr.Length < 6)
{
countStr = "0" + countStr;
}
string fileNameOnly = Path.GetFileNameWithoutExtension(fullPath);
string extension = Path.GetExtension(fullPath);
string path = Path.GetDirectoryName(fullPath);
string newFullPath;
string tempFileName = string.Format("{0}{1}", fileNameOnly, countStr);
newFullPath = Path.Combine(path, tempFileName + extension);
while (File.Exists(newFullPath))
{
count++;
countStr = count.ToString();
while (countStr.Length < 6)
{
countStr = "0" + countStr;
}
tempFileName = string.Format("{0}{1}", fileNameOnly, countStr);
newFullPath = Path.Combine(path, tempFileName + extension);
}
try {
await Task.Run(() => File.WriteAllText(newFullPath, mess.payload));
mess.sent = true;
mess.status = $"Sent file to {newFullPath}";
}
catch (Exception) {
mess.sent = false;
mess.status = $"Unable to write file to {newFullPath}";
}
return(mess);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
await Task.Run(() =>
{
{
if (!OK_TO_SEND)
{
logger.Warn("FTP Interface not configured correctly");
return;
}
string guid = Guid.NewGuid().ToString();
if (!ftpURL.EndsWith("/"))
{
ftpURL = ftpURL + "/";
}
if (!directory.StartsWith("/"))
{
directory = "/" + directory;
}
if (!directory.EndsWith("/"))
{
directory = directory + "/";
}
using (var ftpClient = new FtpClient(new Uri(ftpURL))) {
try {
if (useFTPS)
{
ftpClient.EncryptionMode = FtpEncryptionMode.Explicit;
ftpClient.ValidateAnyCertificate = true;
ftpClient.Connect();
}
ftpClient.Upload(Encoding.UTF8.GetBytes(message.payload), directory + guid);
} catch (Exception ex) {
logger.Error(ex.Message);
}
}
}
});
return(null);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
// Create a new MQTT client.
var msg = new MqttApplicationMessageBuilder()
.WithTopic(topic)
.WithPayload(message.payload)
.WithExactlyOnceQoS()
.WithRetainFlag()
.Build();
//"broker.hivemq.com", 1883)
if (serverType == "tcp")
{
options = new MqttClientOptionsBuilder().WithTcpServer(mqttServer, mqttServerPort).Build();
}
//"broker.hivemq.com:8000/mqtt"
if (serverType == "ws")
{
options = new MqttClientOptionsBuilder().WithWebSocketServer(mqttServerURL).Build();
}
if (!mqttClient.IsConnected)
{
mqttClient.ConnectAsync(options).Wait();
}
MqttClientPublishResult result = await mqttClient.PublishAsync(msg, CancellationToken.None);
if (result.ReasonCode == MqttClientPublishReasonCode.Success)
{
message.sent = true;
message.status = $"Sent to MQTT topic {queueName}";
return(message);
}
else
{
message.sent = true;
message.status = $"MQTT Send Failure {result.ReasonString}";
logger.Error($"MQTT Send Failure {result.ReasonString}");
return(message);
}
}
public void SendToUndeliverableQueue(ExchangeMessage xm)
{
// If messages cant be delivered, they can be sent to a local undeliverable queue
// defined on a per queue basis. No reason why the one undeliverable queue cant be used
// for all of the queue.
if (isLogger)
{
// If this is a logger, failure messages won't be sent to the undeliverable queue
logger.Error("Log Message could not be sent to logger");
return;
}
QXLog(xm?.uuid, "Output Node Send Failure", "Message Could Not Be Delivered to the Output Node", "PROGRESS");
if (undeliverableQueue == null)
{
logger.Debug($"No Undeliverable Message Queue has been defined for {queueName}");
QXLog(xm?.uuid, "Output Node Send Failure", "No Undeliverable Queue Defined", "WARNING");
return;
}
lock (undevlLock) {
try {
using (MessageQueue er = new MessageQueue(undeliverableQueue)) {
QXLog(xm?.uuid, "Output Node Send Failure", "Sending Message to Undeliverable Queue", "PROGRESS");
er.Send(xm);
QXLog(xm?.uuid, "Output Node Send Failure", "Message Sent to Undeliverable Queue", "PROGRESS");
logger.Info("Message Sent to Undeliverble Queue");
}
}
catch (Exception ex) {
QXLog(xm?.uuid, "Output Node Send Failure", "Message Could Not Be Sent To Undeliverable Queue", "ERROR");
logger.Error("Unable to Send to Undeliverble Queue");
logger.Error(ex.StackTrace);
}
}
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage mess)
{
if (maxMessages != -1)
{
MaintainQueueInline();
}
try
{
mess = await SendMqStuff(mess);
if (mess.sent)
{
logger.Debug($"MQ Sent to {name}");
mess.sent = true;
mess.status = $"Sent to {queueName}";
return(mess);
}
else
{
logger.Error($"MQ Failed Sent to {name}");
SendToUndeliverableQueue(mess);
mess.sent = false;
mess.status = $"Unable to Send to {queueName}";
return(mess);
}
}
catch (Exception ex)
{
if (!isLogger)
{
logger.Error(ex, $"MQ Failed Sent to {name}");
}
SendToUndeliverableQueue(mess);
mess.sent = false;
mess.status = $"Unable to Send to {queueName}";
return(mess);
}
}
protected ExchangeMessage SetDestinationFromMessage(ExchangeMessage mess)
{
if (xpathDestination != null)
{
queueName = GetDestinationFromMessage(mess.payload, true);
if (queueName == null)
{
mess.status = "Destination could not be found in the XPATH expression";
mess.destinationSet = false;
return(mess);
}
else
{
mess.destinationSet = true;
return(mess);
}
}
if (xpathContentDestination != null)
{
queueName = GetDestinationFromMessage(mess.payload, false);
if (queueName == null)
{
mess.status = "Destination Topic could not be found in the XPATH expression";
mess.destinationSet = false;
return(mess);
}
else
{
mess.destinationSet = true;
return(mess);
}
}
mess.destinationSet = false;
return(mess);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
await Task.Run(() =>
{
{
if (!OK_TO_SEND)
{
logger.Warn("SMTP Interface not configured correctly");
return;
}
string subject = smtpsubject;
try {
var mailmessage = new MimeMessage();
mailmessage.From.Add(new MailboxAddress(smtpfromUser, smtpfromEmail));
mailmessage.To.Add(new MailboxAddress(smtptoUser, smtptoEmail));
mailmessage.Subject = subject;
if (smtpAttachment)
{
var body = new TextPart("plain")
{
Text = "Message is in attachment"
};
var attachment = new MimePart("text", "plain")
{
ContentDisposition = new ContentDisposition(ContentDisposition.Attachment),
FileName = Path.GetFileName(smtpattachname)
};
byte[] byteArray = Encoding.ASCII.GetBytes(message.payload);
MemoryStream stream = new MemoryStream(byteArray);
attachment.Content = new MimeContent(stream, ContentEncoding.Default);
var multipart = new Multipart("mixed")
{
body,
attachment
};
// now set the multipart/mixed as the message body
mailmessage.Body = multipart;
}
else
{
mailmessage.Body = new TextPart("plain")
{
Text = message.payload
};
}
using (var client = new SmtpClient()) {
// For demo-purposes, accept all SSL certificates (in case the server supports STARTTLS)
client.ServerCertificateValidationCallback = (s, c, h, e) => true;
client.Connect(smtphost, smtpport, smtpuseSSL);
//client.Connect("smtp.mail.yahoo.com", 465, true);
// Note: only needed if the SMTP server requires authentication
if (smtpuser != null)
{
client.Authenticate(smtpuser, smtppass);
}
// client.Authenticate("*****@*****.**", "!@aiw2dihsf!@");
client.Send(mailmessage);
client.Disconnect(true);
}
}
catch (Exception e) {
logger.Error($"SMTP Client Send Exception: {e.Message}");
}
}
});
return(null);
}
public ExchangeMessage PreAndPostProcess(ExchangeMessage xm)
{
QXLog(xm?.uuid, "Output Node: Message Recieved From Pipe", null, "PROGRESS");
// payload is the actual content of the message to be send
string message = xm.payload;
// After the delivery or prior to sending a message, it can be filtered or
// transformed according to the configuration for each queue
// The Expression object manages the evaluation of all the filters
if (expression != null)
{
bool pass = expression.Pass(message);
logger.Info($"Top Expression Evaluated {pass}");
if (!pass)
{
if (altQueue != null)
{
logger.Info($"Sending to Alt Queue {altQueue.name}");
QXLog(xm?.uuid, "Message did not pass filter", "Sending to Alt Queue", "PROGRESS");
Task.Run(async() => { _ = await altQueue.Send(xm); });
}
else
{
QXLog(xm?.uuid, "Message did not pass filter", "No Alt Queue Configured", "WARNING");
}
xm.pass = false;
return(xm);
}
else
{
xm.pass = true;
}
}
if (topLevelFilter != null)
{
bool pass = topLevelFilter.Pass(message);
logger.Info($"Top Filter Evaluated {pass}");
if (!pass)
{
if (altQueue != null)
{
logger.Info($"Sending to Alt Queue {altQueue.name}");
QXLog(xm?.uuid, "Message did not pass filter", "Sending to Alt Queue", "PROGRESS");
_ = altQueue.Send(xm);
}
else
{
QXLog(xm?.uuid, "Message did not pass filter", "No Alt Queue Configured", "WARNING");
}
xm.pass = false;
return(xm);
}
else
{
xm.pass = true;
}
}
// If a XSLT transform has been specified
if (bTransform)
{
QXLog(xm?.uuid, "Starting Message Transformation", null, "PROGRESS");
message = Transform(message, xslVersion);
QXLog(xm?.uuid, "Message Transformation Complete", null, "PROGRESS");
xm.transformed = true;
}
else
{
xm.transformed = false;
}
if (message == null || message.Length == 0)
{
logger.Info("Message blocked by XSL Transform of Zero Length");
xm.payload = null;
xm.status = "Message blocked by XSL Transform. Null or Zero Length";
QXLog(xm?.uuid, "Messaage Transformation", "Transformation resulted in a message of zero length", "WARNING");
return(xm);
}
xm.payload = message;
QXLog(xm?.uuid, "Output Node: Post Processing Complete", null, "PROGRESS");
return(xm);
}
public abstract Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message);
public async Task ContextProcessorAsync(ExchangeMessage xm)
{
if (!xm.pass)
{
return;
}
// If no contextKey has been provided, just inject the message straight away
if (contextCacheKeyXPath == null && !useMessageAsKey)
{
await InjectMessage(xm);
return;
}
string contextKeyValue = GetContextKey(xm);
if (contextKeyValue == null)
{
await InjectMessage(xm);
return;
}
try {
Queue <ExchangeMessage> bufferMemoryQueue = null;
System.Timers.Timer bufferPopperTimer = null;
if (_bufferMemoryQueueDict.ContainsKey(contextKeyValue) && _bufferTimerDict.ContainsKey(contextKeyValue))
{
bufferMemoryQueue = _bufferMemoryQueueDict[contextKeyValue];
bufferPopperTimer = _bufferTimerDict[contextKeyValue];
}
else
{
// They don't exist, so they need to be created.
logger.Trace($"Creating queue and timer for {contextKeyValue}");
//Create the queue
bufferMemoryQueue = new Queue <ExchangeMessage>();
_bufferMemoryQueueDict.Add(contextKeyValue, bufferMemoryQueue);
//Create the timer
bufferPopperTimer = CreatePopperTask(xm, bufferMemoryQueue, contextKeyValue);
_bufferTimerDict.Add(contextKeyValue, bufferPopperTimer);
}
ContextStats stats = null;
if (statDict.ContainsKey(contextKeyValue))
{
stats = statDict[contextKeyValue];
}
else
{
ContextStats v = new ContextStats {
key = contextKeyValue
};
statDict.Add(contextKeyValue, v);
stats = v;
}
stats.recieved++;
this.totalReceived++;
if (_contextCache.Contains(contextKeyValue) && this.discardInCache)
{
logger.Info($"Message found in Cache - Discarding. Message Hash {contextKeyValue}");
return;
}
if (mostRecentOnly)
{
bufferMemoryQueue.Clear();
}
bufferMemoryQueue.Enqueue(xm);
// So the message is enqueued, now just work out how long how to set the popper
//Let's deal with it if it is not a firstOnly
if (!firstOnly)
{
if (!_contextCache.Contains(contextKeyValue))
{
//It's not in the cachce
if (!bufferPopperTimer.Enabled)
{
//If the popper isn't already enabled, enable it to pop straight away (5ms)
bufferPopperTimer.Interval = 5.0;
bufferPopperTimer.Enabled = true;
bufferPopperTimer.Start();
}
}
else
{
//It's in tha cache,
if (!bufferPopperTimer.Enabled)
{
//If the popper isn't already enabled, then set the popper. (If it is in the cache, then really, the popper should already be set)
bufferPopperTimer.Interval = contextCacheExpiry * 1000;
bufferPopperTimer.Enabled = true;
bufferPopperTimer.Start();
}
}
return;
}
if (firstOnly)
{
//It's not in the cache
bool inCache = _contextCache.Contains(contextKeyValue);
bool firstDone = _firstProcessedCompleted.Contains(contextKeyValue);
if (!inCache || firstDone)
{
_contextCache.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));
if (!bufferPopperTimer.Enabled)
{
bufferPopperTimer.Interval = 10.0;
bufferPopperTimer.Enabled = true;
bufferPopperTimer.Start();
}
}
else
{
if (!bufferPopperTimer.Enabled)
{
bufferPopperTimer.Interval = contextCacheExpiry * 1000;
bufferPopperTimer.Enabled = true;
bufferPopperTimer.Start();
}
}
return;
}
} catch (Exception ex) {
logger.Error(ex);
}
}
public new async void Send(ExchangeMessage xm)
{
logger.Debug($"Sending to {xm.uuid} IBMMQ {queueName}");
await SendToOutputAsync(xm);
}
//public override ExchangeMessage Listen(bool immediateReturn, int priorityWait)
//{
// // The serviceQueue is monitored and messages are returned as the appear on the queue
// logger.Debug("Listening to KAFKA QUEUE");
// return (serviceQueue.Listen(immediateReturn, priorityWait));
//}
public new async void Send(ExchangeMessage xm)
{
logger.Debug($"Sending to {xm.uuid} Kafka Topic {topic}");
await SendToOutputAsync(xm);
}
public override async Task <ExchangeMessage> SendToOutputAsync(ExchangeMessage message)
{
await Task.Run(() => { });
// Set the queueName bassed on the content of the message if configured
message = SetDestinationFromMessage(message);
if (!message.destinationSet && queueName == null)
{
message.sent = false;
return(message);
}
try {
var factory = new ConnectionFactory()
{
HostName = connection
};
factory.UserName = user;
factory.Password = pass;
factory.VirtualHost = vhost;
factory.HostName = connection;
factory.Port = port;
using (var conn = factory.CreateConnection()) {
using (var channel = conn.CreateModel()) {
try {
QueueDeclareOk ch = channel.QueueDeclare(queue: queueName,
durable: true,
exclusive: false,
autoDelete: false,
arguments: null);
} catch (Exception ex) {
logger.Error(ex.Message);
logger.Error(ex.StackTrace);
message.sent = false;
message.status = $"Error Sending to Rabbit Topic {queueName}";
return(message);
}
var body = Encoding.UTF8.GetBytes(message.payload);
try {
channel.BasicPublish(exchange: "",
routingKey: queueName,
basicProperties: null,
body: body);
} catch (Exception ex) {
logger.Error(ex.Message);
logger.Error(ex.StackTrace);
SendToUndeliverableQueue(message);
message.sent = false;
message.status = $"Error Sending to Rabbit Queue {queueName}";
return(message);
}
message.sent = true;
message.status = $"Sent to Rabbit Queue {queueName}";
return(message);
}
}
} catch (Exception e) {
logger.Error(e.Message);
message.sent = false;
return(message);
}
}
private System.Timers.Timer CreatePopperTask(ExchangeMessage xm, Queue <ExchangeMessage> bufferMemoryQueue, string contextKeyValue)
{
double interval = contextCacheExpiry * 1000 + 100.0;
logger.Info($"Creating Popper Task. Message Hash: {contextKeyValue}");
System.Timers.Timer bufferPopperTimer = new System.Timers.Timer {
Interval = interval,
Enabled = false
};
bufferPopperTimer.Elapsed += async(source, eventArgs) =>
{
bufferPopperTimer.AutoReset = false;
bufferPopperTimer.Enabled = false;
if (_firstProcessedCompleted.Contains(contextKeyValue))
{
// logger.Info("Pooper Expired - Marking it cleared <<<<<<<<<<<<<<<<<<");
_firstProcessedCleared.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));
}
if (bufferMemoryQueue.Count != 0)
{
logger.Trace($"Injection Popper Timer Went Off for key = {contextKeyValue} - {bufferMemoryQueue.Count} Messages on queue");
_firstProcessedCompleted.AddOrGetExisting(contextKeyValue, contextKeyValue, DateTime.Now.AddHours(18));
_contextCache.AddOrGetExisting(contextKeyValue, DateTime.Now.AddSeconds(this.contextCacheExpiry), DateTime.Now.AddSeconds(this.contextCacheExpiry));
if (bufferMemoryQueue.Count > 0)
{
await InjectMessage(bufferMemoryQueue.Dequeue());
}
else
{
return;
}
if (firstOnly && _firstProcessedCleared.Contains(contextKeyValue))
{
bufferPopperTimer.Interval = 10.0;
}
else
{
bufferPopperTimer.Interval = contextCacheExpiry * 1000;
}
bufferPopperTimer.AutoReset = true;
bufferPopperTimer.Enabled = true;
try {
statDict[contextKeyValue].send++;
statLogger.Trace(statDict[contextKeyValue]);
} catch (Exception ex) {
logger.Error($"Stats dictionary problem {ex.Message}");
}
this.totalSent++;
}
else
{
logger.Trace($"Reinjection Popper Timer Went Off for key = {contextKeyValue} - No Message to Process");
if (_firstProcessedCleared.Contains(contextKeyValue))
{
bufferPopperTimer.Enabled = false;
bufferPopperTimer.AutoReset = false;
}
else
{
bufferPopperTimer.Interval = contextCacheExpiry * 1000;
bufferPopperTimer.Enabled = true;
bufferPopperTimer.AutoReset = true;
}
}
};
return(bufferPopperTimer);
}
protected async Task InjectMessage(ExchangeMessage xm)
{
if (xm != null)
{
if (!xm.pass)
{
logger.Trace($"Blocked by incoming filter. Message {xm.uuid }");
return;
}
msgRecieved++;
logger.Trace($"Processing message {xm.uuid } [{msgRecieved}]");
xm.status = "Message Recieved";
// Pipeline specific logging
QXLog("Starting Output Meesage Processing", null, "PROGRESS");
if (randomDistribution)
{
// Distributes the message to a random output
Random rnd = new Random();
int index = rnd.Next(0, output.Count);
logger.Trace($"Random Distributor. Sending message {xm.uuid } to {output[index].name}");
QXLog("Output Meesage Processing", "Distributing to Random Output", "PROGRESS");
if (outputIsolation)
{
QXLog("Async Message Start", "Distribution to Random Output", "PROGRESS");
_ = Task.Run(() => SendAndLog(output[index], xm));
QXLog("Async Message End", "Distribution to Random Output", "PROGRESS");
}
else
{
QXLog("Sync Message Start", "Distribution to Random Output", "PROGRESS");
_ = await SendAndLog(output[index], xm);
QXLog("Sync Message End", "Distribution to Random Output", "PROGRESS");
}
}
else if (roundRobinDistribution)
{
// Distributes the message only once to each queue in turn
int index = nextOutput % output.Count;
logger.Trace($"Round Robin Distributor. Sending message {xm.uuid } to {output[index].name}");
QXLog("Output Message Processing", "Distribution to Round Robin Output", "PROGRESS");
if (outputIsolation)
{
QXLog("Async Message Start", "Distribution to Round Robin Output", "PROGRESS");
_ = Task.Run(() => SendAndLog(output[index], xm));
QXLog("Async Message End", "Distribution to Round Robin Output", "PROGRESS");
}
else
{
QXLog("Sync Message Start", "Distribution to Round Robin Output", "PROGRESS");
_ = await SendAndLog(output[index], xm);
QXLog("Sync Message End", "Distribution to Round Robin Output", "PROGRESS");
}
nextOutput++;
}
else
{
// Distribute the message to all the outputs
if (outputIsolation)
{
// Wont wait for the Send to complete before completing
foreach (QueueAbstract q in output)
{
QXLog("Async Message Start", "Normal Distribution", "PROGRESS");
_ = Task.Run(() => SendAndLog(q, xm));
QXLog("Async Message End", "Normal Distribution", "PROGRESS");
}
}
else
{
// Will wait for all Sends to complete before proceeding
QXLog("Sync Message Start", "Normal Distribution", "PROGRESS");
foreach (QueueAbstract q in output)
{
logger.Info($"-->{name} --> sending message {xm?.uuid } to {q.name}");
_ = await SendAndLog(q, xm);
//if (xm.sent)
//{
// if (q.queueName != null)
// {
// msgCount[q.queueName] = msgCount[q.queueName] + 1;
// logger.Trace($"{q.queueName} = {msgCount[q.queueName]}");
// }
// else
// {
// msgCount[q.name] = msgCount[q.name] + 1;
// logger.Trace($"{q.name} = {msgCount[q.name]}");
// }
//}
}
QXLog("Sync Message End", "Normal Distribution", "PROGRESS");
}
}
}
}
private async Task <ExchangeMessage> SendAndLog(QueueAbstract queue, ExchangeMessage message)
{
message = await queue.Send(message);
return(message);
}
