protected override async Task ExecuteAsync(CancellationToken cancellationToken)
{
_client = new AmazonSQSClient(_globalSettings.Amazon.AccessKeyId,
_globalSettings.Amazon.AccessKeySecret, RegionEndpoint.GetBySystemName(_globalSettings.Amazon.Region));
var blockIpQueue = await _client.GetQueueUrlAsync("block-ip", cancellationToken);
var blockIpQueueUrl = blockIpQueue.QueueUrl;
var unblockIpQueue = await _client.GetQueueUrlAsync("unblock-ip", cancellationToken);
var unblockIpQueueUrl = unblockIpQueue.QueueUrl;
while (!cancellationToken.IsCancellationRequested)
{
var blockMessageResponse = await _client.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = blockIpQueueUrl,
MaxNumberOfMessages = 10,
WaitTimeSeconds = 15
}, cancellationToken);
if (blockMessageResponse.Messages.Any())
{
foreach (var message in blockMessageResponse.Messages)
{
try
{
await BlockIpAsync(message.Body, cancellationToken);
}
catch (Exception e)
{
_logger.LogError(e, "Failed to block IP.");
}
await _client.DeleteMessageAsync(blockIpQueueUrl, message.ReceiptHandle, cancellationToken);
}
}
var unblockMessageResponse = await _client.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = unblockIpQueueUrl,
MaxNumberOfMessages = 10,
WaitTimeSeconds = 15
}, cancellationToken);
if (unblockMessageResponse.Messages.Any())
{
foreach (var message in unblockMessageResponse.Messages)
{
try
{
await UnblockIpAsync(message.Body, cancellationToken);
}
catch (Exception e)
{
_logger.LogError(e, "Failed to unblock IP.");
}
await _client.DeleteMessageAsync(unblockIpQueueUrl, message.ReceiptHandle, cancellationToken);
}
}
await Task.Delay(TimeSpan.FromSeconds(15));
}
}
#pragma warning disable 4014
private void SubscriberThread()
{
if (sqlController.SettingRead(Settings.token).Result != "UNIT_TEST___________________L:32")
#region amazon
{
log.LogStandard(t.GetMethodName("Subscriber"), $"{DateTime.Now.ToString()} - Starting up");
#region setup
isActive = true;
keepSubscribed = true;
string awsAccessKeyId = sqlController.SettingRead(Settings.awsAccessKeyId).Result;
string awsSecretAccessKey = sqlController.SettingRead(Settings.awsSecretAccessKey).Result;
string awsQueueUrl = sqlController.SettingRead(Settings.awsEndPoint).Result + sqlController.SettingRead(Settings.token).Result;
var sqsClient = new AmazonSQSClient(awsAccessKeyId, awsSecretAccessKey, RegionEndpoint.EUCentral1);
DateTime lastException = DateTime.MinValue;
#endregion
while (keepSubscribed)
{
try
{
var res = sqsClient.ReceiveMessageAsync(awsQueueUrl).Result;
if (res.Messages.Count > 0)
{
foreach (var message in res.Messages)
{
#region JSON -> var
var parsedData = JRaw.Parse(message.Body);
string notificationUId = parsedData["id"].ToString();
int microtingUId = int.Parse(parsedData["microting_uuid"].ToString());
string action = parsedData["text"].ToString();
#endregion
log.LogStandard(t.GetMethodName("Subscriber"),
"Notification notificationUId : " + notificationUId + " microtingUId : " +
microtingUId + " action : " + action);
notifications result;
switch (action)
{
case Constants.Notifications.Completed:
result = sqlController.NotificationCreate(notificationUId, microtingUId,
Constants.Notifications.Completed).Result;
bus.SendLocal(new EformCompleted(notificationUId, microtingUId));
break;
case Constants.Notifications.EformParsedByServer:
bus.SendLocal(new EformParsedByServer(notificationUId, microtingUId));
break;
case Constants.Notifications.EformParsingError:
bus.SendLocal(new EformParsingError(notificationUId, microtingUId));
break;
case Constants.Notifications.RetrievedForm:
result = sqlController.NotificationCreate(notificationUId, microtingUId,
Constants.Notifications.RetrievedForm).Result;
bus.SendLocal(new EformRetrieved(notificationUId, microtingUId));
break;
case Constants.Notifications.UnitActivate:
result = sqlController.NotificationCreate(notificationUId, microtingUId,
Constants.Notifications.UnitActivate).Result;
bus.SendLocal(new UnitActivated(notificationUId, microtingUId));
break;
case Constants.Notifications.SpeechToTextCompleted:
result = sqlController.NotificationCreate(notificationUId, microtingUId,
Constants.Notifications.SpeechToTextCompleted).Result;
bus.SendLocal(new TranscriptionCompleted(notificationUId, microtingUId));
break;
case Constants.Notifications.InSightAnswerDone:
result = sqlController.NotificationCreate(notificationUId, microtingUId,
Constants.Notifications.InSightAnswerDone).Result;
bus.SendLocal(new AnswerCompleted(notificationUId, microtingUId));
break;
}
sqsClient.DeleteMessageAsync(awsQueueUrl, message.ReceiptHandle);
}
}
else
{
log.LogStandard(t.GetMethodName("Subscriber"),
$"{DateTime.Now.ToString()} - No messages for us right now!");
}
}
catch (WebException webException)
{
log.LogWarning(t.GetMethodName("Subscriber"), t.PrintException(t.GetMethodName("Subscriber") + " failed", webException));
// We try to sleep 20 seconds to see if the problem goes away by it self.
Thread.Sleep(20000);
}
catch (Exception ex)
{
// Log exception
log.LogWarning(t.GetMethodName("Subscriber"), t.PrintException(t.GetMethodName("Subscriber") + " failed", ex));
if (DateTime.Compare(lastException.AddMinutes(5), DateTime.Now) > 0)
{
keepSubscribed = false;
log.LogException(t.GetMethodName("Subscriber"), "failed, twice in the last 5 minuts", ex);
// TODO handle crash so we could restart!!!
}
lastException = DateTime.Now;
}
}
log.LogStandard(t.GetMethodName("Subscriber"), "--- WE WHERE TOLD NOT TO CONTINUE TO SUBSCRIBE ---");
sqsClient.Dispose();
//EventMsgClient("Subscriber closed", null);
keepSubscribed = false;
isActive = false;
}
#endregion
else
#region unit test
{
log.LogStandard(t.GetMethodName("Subscriber"), "Subscriber faked");
isActive = true;
keepSubscribed = true;
while (keepSubscribed)
{
Thread.Sleep(100);
}
keepSubscribed = false;
isActive = false;
}
#endregion
}
/// <summary>
/// Receives the specified queue name.
/// </summary>
/// <param name="timeoutInMilliseconds">The timeout
public Message[] Receive(int timeoutInMilliseconds)
{
AmazonSQSClient client = null;
Amazon.SQS.Model.Message sqsMessage = null;
try
{
client = new AmazonSQSClient(_connection.Credentials, _connection.Region);
var urlResponse = client.GetQueueUrlAsync(_queueName).Result;
_logger.Value.DebugFormat("SqsMessageConsumer: Preparing to retrieve next message from queue {0}",
urlResponse.QueueUrl);
var request = new ReceiveMessageRequest(urlResponse.QueueUrl)
{
MaxNumberOfMessages = _batchSize,
WaitTimeSeconds = (int)TimeSpan.FromMilliseconds(timeoutInMilliseconds).TotalSeconds,
MessageAttributeNames = new List <string>()
{
"All"
},
AttributeNames = new List <string>()
{
"All"
}
};
var receiveResponse = client.ReceiveMessageAsync(request).Result;
sqsMessage = receiveResponse.Messages.SingleOrDefault();
}
catch (InvalidOperationException ioe)
{
_logger.Value.DebugFormat("SqsMessageConsumer: Could not determine number of messages to retrieve");
throw new ChannelFailureException("Error connecting to SQS, see inner exception for details", ioe);
}
catch (OperationCanceledException oce)
{
_logger.Value.DebugFormat("SqsMessageConsumer: Could not find messages to retrieve");
throw new ChannelFailureException("Error connecting to SQS, see inner exception for details", oce);
}
catch (Exception e)
{
_logger.Value.ErrorException("SqsMessageConsumer: There was an error listening to queue {0} ", e, _queueName);
throw;
}
finally
{
client?.Dispose();
}
if (sqsMessage == null)
{
return new Message[] { _noopMessage }
}
;
var message = new SqsMessageCreator().CreateMessage(sqsMessage);
_logger.Value.InfoFormat("SqsMessageConsumer: Received message from queue {0}, message: {1}{2}",
_queueName, Environment.NewLine, JsonConvert.SerializeObject(message));
return(new Message[] { message });
}
/// <summary>
/// Deletes all messages from the input queue (which is done by receiving them in batches and deleting them, as long as it takes)
/// </summary>
public void Purge()
{
if (Address == null)
{
return;
}
_log.Info("Purging {0} (by receiving all messages from the queue)", Address);
try
{
using (var client = new AmazonSQSClient(_credentials, _amazonSqsConfig))
{
var stopwatch = Stopwatch.StartNew();
while (true)
{
var receiveTask = client.ReceiveMessageAsync(new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 10
});
AsyncHelpers.RunSync(() => receiveTask);
var response = receiveTask.Result;
if (!response.Messages.Any())
{
break;
}
var deleteTask = client.DeleteMessageBatchAsync(_queueUrl, response.Messages
.Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle))
.ToList());
AsyncHelpers.RunSync(() => deleteTask);
var deleteResponse = deleteTask.Result;
if (deleteResponse.Failed.Any())
{
var errors = string.Join(Environment.NewLine,
deleteResponse.Failed.Select(f => $"{f.Message} ({f.Id})"));
throw new RebusApplicationException(
$@"Error {deleteResponse.HttpStatusCode} while purging: {errors}");
}
}
_log.Info($"Purging {Address} took {stopwatch.Elapsed.TotalSeconds:0.0} s");
}
}
catch (AmazonSQSException exception) when(exception.StatusCode == HttpStatusCode.BadRequest)
{
if (exception.Message.Contains("queue does not exist"))
{
return;
}
throw;
}
catch (Exception exception)
{
throw new RebusApplicationException(exception, $"Error while purging {Address}");
}
}
public static async Task <GlacierResult> ProcessQueueAsync(Topic topic)
{
// Check for notifications on topic and process any message
try
{
var settings = GetSettingsAsync().Result;
using (var client = new AmazonGlacierClient(
settings.AWSAccessKeyID,
settings.AWSSecretAccessKey,
RegionEndpoint.GetBySystemName(settings.AWSS3Region.SystemName)))
{
var receiveMessageRequest = new ReceiveMessageRequest {
QueueUrl = topic.QueueUrl, MaxNumberOfMessages = 1
};
var sqsClient = new AmazonSQSClient(settings.AWSAccessKeyID, settings.AWSSecretAccessKey, RegionEndpoint.GetBySystemName(settings.AWSS3Region.SystemName));
var receiveMessageResponse = await sqsClient.ReceiveMessageAsync(receiveMessageRequest);
if (receiveMessageResponse.Messages.Count == 0)
{
topic.Status = GlacierResult.Incomplete;
SaveTopicFile(topic);
return(topic.Status);
}
// Process message
string status = GetResponseStatus(receiveMessageResponse);
if (string.Equals(status, GlacierUtils.JOB_STATUS_SUCCEEDED, StringComparison.OrdinalIgnoreCase))
{
DownloadGlacierJobOutputAsync(topic.JobId, client, settings.AWSGlacierVault, topic.GetOutputFile());
Debug.WriteLine($"Downloaded job output to {topic.GetOutputFile()}");
if (topic.ArchiveId != null)
{
DownloadSuccess?.Invoke($"Glacier archive was downloaded to {topic.GetOutputFile()}");
}
await DeleteTopicAsync(topic);
return(GlacierResult.Completed);
}
else if (string.Equals(status, GlacierUtils.JOB_STATUS_FAILED, StringComparison.OrdinalIgnoreCase))
{
DownloadError?.Invoke("Job failed, cannot download the file");
await DeleteTopicAsync(topic);
return(GlacierResult.JobFailed);
}
else if (string.Equals(status, GlacierUtils.JOB_STATUS_INPROGRESS, StringComparison.OrdinalIgnoreCase))
{
DownloadWarning?.Invoke("Job in progress, Queue ARN: " + topic.QueueARN);
await DeleteTopicAsync(topic);
return(GlacierResult.JobInProgress);
}
else
{
await DeleteTopicAsync(topic);
return(GlacierResult.Error);
}
}
}
catch (AmazonServiceException azex)
{
// Handle specific potential errors here
Debug.WriteLine("AmazonServiceException " + azex.Message);
if (azex.StatusCode == System.Net.HttpStatusCode.Forbidden)
{
// Invalid credentials
BackupError?.Invoke("Invalid AWS credentials were provided while connecting");
return(GlacierResult.Incomplete);
}
if (azex.InnerException != null &&
azex.InnerException is System.Net.WebException &&
((System.Net.WebException)azex.InnerException).Status == System.Net.WebExceptionStatus.NameResolutionFailure)
{
// Not connected to internet
BackupError?.Invoke("Network connection failure");
return(GlacierResult.Incomplete);
}
if (azex.InnerException != null &&
azex.InnerException is System.Net.WebException)
{
// Network errors
BackupError?.Invoke($"A network error occurred ({((System.Net.WebException)azex.InnerException).Status})");
return(GlacierResult.Incomplete);
}
if (azex.StatusCode == System.Net.HttpStatusCode.BadRequest
//&& topic.Status == GlacierResult.JobRequested
&& azex.Message.Contains("The specified queue does not exist") &&
DateTime.Now - topic.DateRequested < new TimeSpan(24, 0, 0))
{
// Job was recently requested and the queue has not been created yet
Debug.WriteLine("Job request may be in progress");
return(GlacierResult.JobRequested);
}
// TODO Check expiry?
// Glacier ref: "A job ID will not expire for at least 24 hours after Amazon Glacier completes the job."
await DeleteTopicAsync(topic);
BackupWarning?.Invoke("An AWS Glacier job has expired, a new job will be issued");
// Reissue expired job
InitiateGlacierJob(topic);
return(topic.Status);
}
catch (Exception ex)
{
await DeleteTopicAsync(topic);
throw ex;
}
}
public async Task <ReceiveMessageResponse> ReceiveMessageAsync(string queueUrl)
{
return(await sqsClient.ReceiveMessageAsync(queueUrl));
}
private async Task Handle(string queueUrl, CancellationToken cancellationToken)
{
using var correlationIdScope = _logger.BeginScope(
new Dictionary <string, object>
{
["correlation_id"] = Guid.NewGuid(),
["queue_url"] = queueUrl
});
_logger.LogTrace("Start polling messages from a queue: {queue_url}. correlation_id: {correlation_id}");
ReceiveMessageResponse receiveMessageResult = new ReceiveMessageResponse();
try
{
receiveMessageResult = await _amazonSqsClient
.ReceiveMessageAsync(new ReceiveMessageRequest
{
WaitTimeSeconds = _config.WaitTimeSeconds,
MaxNumberOfMessages = _config.MaxNumberOfMessages,
MessageAttributeNames = _config.MessageAttributeNames,
QueueUrl = queueUrl
}, cancellationToken);
var messagesCount = receiveMessageResult.Messages.Count;
_logger.LogTrace("{count} messages received", messagesCount);
foreach (var message in receiveMessageResult.Messages)
{
using var messageIdScope = _logger.BeginScope(
new Dictionary <string, object>
{
["message_id"] = message.MessageId,
["receipt_handle"] = message.ReceiptHandle
});
_logger.LogTrace(
"Start processing the message with id {message_id} and ReceiptHandle {receipt_handle}");
#pragma warning disable 4014
_consumerResolver
.Resolve(message, cancellationToken)
.ContinueWith(task =>
#pragma warning restore 4014
{
if (task.IsFaulted)
{
_logger.LogError(task.Exception,
"Failed to handle message with id {message_id} and ReceiptHandle {receipt_handle}");
return;
}
_logger.LogTrace(
"Deleting the message with id {message_id} and ReceiptHandle {receipt_handle}");
_amazonSqsClient
.DeleteMessageAsync(new DeleteMessageRequest
{
QueueUrl = queueUrl,
ReceiptHandle = message.ReceiptHandle
}, cancellationToken)
.ContinueWith(deleteMessageTask =>
{
if (deleteMessageTask.IsFaulted)
{
_logger.LogError(task.Exception,
"Failed to deleete message with id {message_id} and ReceiptHandle {receipt_handle}");
return;
}
_logger.LogTrace(
"The message with id {message_id} and ReceiptHandle {receipt_handle} has been deleted successfully");
}, cancellationToken);
}, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to receive messages from the queue");
}
}
/// <summary>
/// Receives the specified queue name.
/// </summary>
/// <param name="timeoutInMilliseconds">The timeout in milliseconds. Anytyhing greater than 0 uses long-polling </param>
public Message[] Receive(int timeoutInMilliseconds)
{
AmazonSQSClient client = null;
Amazon.SQS.Model.Message[] sqsMessages;
try
{
client = new AmazonSQSClient(_awsConnection.Credentials, _awsConnection.Region);
var urlResponse = client.GetQueueUrlAsync(_queueName).GetAwaiter().GetResult();
s_logger.LogDebug("SqsMessageConsumer: Preparing to retrieve next message from queue {0}",
urlResponse.QueueUrl);
var request = new ReceiveMessageRequest(urlResponse.QueueUrl)
{
MaxNumberOfMessages = _batchSize,
WaitTimeSeconds = (int)TimeSpan.FromMilliseconds(timeoutInMilliseconds).TotalSeconds,
MessageAttributeNames = new List <string>()
{
"All"
},
AttributeNames = new List <string>()
{
"All"
}
};
var receiveResponse = client.ReceiveMessageAsync(request).GetAwaiter().GetResult();
sqsMessages = receiveResponse.Messages.ToArray();
}
catch (InvalidOperationException ioe)
{
s_logger.LogDebug("SqsMessageConsumer: Could not determine number of messages to retrieve");
throw new ChannelFailureException("Error connecting to SQS, see inner exception for details", ioe);
}
catch (OperationCanceledException oce)
{
s_logger.LogDebug("SqsMessageConsumer: Could not find messages to retrieve");
throw new ChannelFailureException("Error connecting to SQS, see inner exception for details", oce);
}
catch (Exception e)
{
s_logger.LogError(e, "SqsMessageConsumer: There was an error listening to queue {0} ", _queueName);
throw;
}
finally
{
client?.Dispose();
}
if (sqsMessages.Length == 0)
{
return(new[] { _noopMessage });
}
var messages = new Message[sqsMessages.Length];
for (int i = 0; i < sqsMessages.Length; i++)
{
var message = new SqsMessageCreator().CreateMessage(sqsMessages[i]);
s_logger.LogInformation("SqsMessageConsumer: Received message from queue {0}, message: {1}{2}",
_queueName, Environment.NewLine, JsonSerializer.Serialize(message, JsonSerialisationOptions.Options));
messages[i] = message;
}
return(messages);
}
private async Task Handle(CancellationToken cancellationToken)
{
using (_logger.BeginScope(new Dictionary <string, object>
{
["correlation_id"] = Guid.NewGuid()
}))
{
_logger.LogDebug("Start pooling messages from a queue. correlation_id: {correlation_id}");
ReceiveMessageResponse receiveMessageResult = null;
try
{
receiveMessageResult = await _amazonSqsClient
.ReceiveMessageAsync(new ReceiveMessageRequest
{
WaitTimeSeconds = _config.WaitTimeSeconds,
MaxNumberOfMessages = _config.MaxNumberOfMessages,
MessageAttributeNames = _config.MessageAttributeNames,
QueueUrl = _config.QueueUrl
}, cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to receive messages from the queue");
}
var messagesCount = receiveMessageResult.Messages.Count;
_logger.LogDebug("{count} messages received", messagesCount);
foreach (var msg in receiveMessageResult.Messages)
{
try
{
var messageType = msg.MessageAttributes
.FirstOrDefault(x => x.Key == "MessageType").Value?.StringValue;
using (_logger.BeginScope(new Dictionary <string, object>
{
["message_type"] = messageType
}))
{
if (messageType != null)
{
_logger.LogDebug("Message Type is {message_type}");
_consumerResolver.Resolve(msg.Body, messageType, cancellationToken);
}
else
{
var body = JsonConvert.DeserializeObject <MessageBody>(msg.Body);
messageType = body.MessageAttributes
.FirstOrDefault(x => x.Key == "MessageType").Value.Value;
_logger.LogDebug("Message Type is {message_type}");
_consumerResolver.Resolve(body.Message, messageType, cancellationToken);
}
}
_logger.LogDebug("Deleting the message {message_id}", msg.ReceiptHandle);
await _amazonSqsClient.DeleteMessageAsync(new DeleteMessageRequest
{
QueueUrl = _config.QueueUrl,
ReceiptHandle = msg.ReceiptHandle
}, cancellationToken);
_logger.LogDebug(
"The message {message_id} has been deleted successfully",
msg.ReceiptHandle);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
catch (Exception ex)
{
_logger.LogError(
"Failed to handle message {message_id}. {@ex}", msg.ReceiptHandle, ex);
}
}
}
}
public async Task SetQueueConfigurationTests()
{
var filterRule = new FilterRule("Prefix", "test/");
using (var sqsClient = new AmazonSQSClient())
{
string topicName = UtilityMethods.GenerateName("events-test");
var createResponse = await sqsClient.CreateQueueAsync(topicName);
var bucketName = await UtilityMethods.CreateBucketAsync(Client, "SetQueueConfigurationTests");
try
{
var queueArn = await sqsClient.AuthorizeS3ToSendMessageAsync(createResponse.QueueUrl, bucketName);
PutBucketNotificationRequest putRequest = new PutBucketNotificationRequest
{
BucketName = bucketName,
QueueConfigurations = new List <QueueConfiguration>
{
new QueueConfiguration
{
Id = "the-queue-test",
Queue = queueArn,
Events = { EventType.ObjectCreatedPut },
Filter = new Filter
{
S3KeyFilter = new S3KeyFilter
{
FilterRules = new List <FilterRule>
{
filterRule
}
}
}
}
}
};
await Client.PutBucketNotificationAsync(putRequest);
var getResponse = WaitUtils.WaitForComplete(
() =>
{
return(Client.GetBucketNotificationAsync(bucketName).Result);
},
(r) =>
{
return(r.QueueConfigurations.Count > 0);
});
Assert.Equal(1, getResponse.QueueConfigurations.Count);
Assert.Equal(1, getResponse.QueueConfigurations[0].Events.Count);
Assert.Equal(EventType.ObjectCreatedPut, getResponse.QueueConfigurations[0].Events[0]);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules);
Assert.Equal(1, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules.Count);
Assert.Equal(filterRule.Name, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Name);
Assert.Equal(filterRule.Value, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Value);
Assert.Equal("the-queue-test", getResponse.QueueConfigurations[0].Id);
Assert.Equal(queueArn, getResponse.QueueConfigurations[0].Queue);
// Purge queue to remove test message sent configuration was setup.
await sqsClient.PurgeQueueAsync(createResponse.QueueUrl);
// make sure the queue is really clear
WaitUtils.WaitForComplete(
() =>
{
return(sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest()
{
QueueUrl = createResponse.QueueUrl
}).Result);
},
(r) =>
{
return(r.Messages.Count == 0);
});
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = "test/data.txt",
ContentBody = "Important Data"
};
await Client.PutObjectAsync(putObjectRequest);
var response = WaitUtils.WaitForComplete(
() =>
{
return(sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = createResponse.QueueUrl,
WaitTimeSeconds = 20
}).Result);
},
(r) =>
{
return(r.Messages.Count > 0);
});
var evnt = S3EventNotification.ParseJson(response.Messages[0].Body);
Assert.Equal(1, evnt.Records.Count);
Assert.Equal(putObjectRequest.BucketName, evnt.Records[0].S3.Bucket.Name);
Assert.Equal(putObjectRequest.Key, evnt.Records[0].S3.Object.Key);
Assert.Equal(putObjectRequest.ContentBody.Length, evnt.Records[0].S3.Object.Size);
}
finally
{
await sqsClient.DeleteQueueAsync(createResponse.QueueUrl);
await UtilityMethods.DeleteBucketWithObjectsAsync(Client, bucketName);
}
}
}
static void Main(string[] args)
{
Console.WriteLine("Initializing Amazon SQS Client and getting queue URL...");
var sqsClient = new AmazonSQSClient();
var getQueueUrlRequest = new GetQueueUrlRequest
{
QueueName = "DemoMasterWorkerPattern",
QueueOwnerAWSAccountId = "545983628851"
};
var getQueueUrlResponse = sqsClient.GetQueueUrlAsync(getQueueUrlRequest).Result;
var queueUrl = getQueueUrlResponse.QueueUrl;
Console.WriteLine($"Received Queue URL: {queueUrl}");
var numberOfParallelTasks = 10;
Console.WriteLine($"Spawning {numberOfParallelTasks} to receive and process the messages");
var cts = new CancellationTokenSource();
var cancellationToken = cts.Token;
// var task = Task.Run(() => {
Parallel.For(1, numberOfParallelTasks + 1, i => {
Console.WriteLine($"Task {i}: Reading messages from queue.");
while (true)
{
var receiveMessageRequest = new ReceiveMessageRequest
{
QueueUrl = queueUrl,
MaxNumberOfMessages = 1,
WaitTimeSeconds = 20
};
var receiveMessageResponse = sqsClient.ReceiveMessageAsync(receiveMessageRequest).Result;
Console.WriteLine($"Task {i}: Received {receiveMessageResponse.Messages.Count} messages.");
if (receiveMessageResponse.Messages.Count != 0)
{
var message = receiveMessageResponse.Messages[0];
Console.WriteLine($"Task {i}: Processing messages...");
Console.WriteLine($"Task {i}: Publisher said: '{message.Body}'");
Thread.Sleep(2000);
Console.WriteLine($"Task {i}: Deleting message from the queue...");
sqsClient.DeleteMessageAsync(queueUrl, message.ReceiptHandle);
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
}
});
// }, cancellationToken);
// Console.WriteLine("Running... Press any key to exit");
// Console.ReadLine();
// Console.WriteLine("Canceling gracefully...");
// cts.Cancel();
// while (!task.IsCompleted)
// {
// Thread.Sleep(1000);
// }
}
public async void TestQueueSubscription()
{
var topicArn = await CreateTopic();
var queueUrl = await CreateQueue();
var subscriptionArn = await SubscribeQueue(topicArn, queueUrl);
var publishRequest = GetPublishRequest(topicArn);
List <Message> messages = await PublishToSNSAndReceiveMessages(publishRequest, topicArn, queueUrl);
Assert.Equal(1, messages.Count);
var message = messages[0];
string bodyJson = GetBodyJson(message);
var json = ThirdParty.Json.LitJson.JsonMapper.ToObject(bodyJson);
var messageText = json["Message"];
var messageSubject = json["Subject"];
Assert.Equal(publishRequest.Message, messageText.ToString());
Assert.Equal(publishRequest.Subject, messageSubject.ToString());
var messageAttributes = json["MessageAttributes"];
Assert.Equal(publishRequest.MessageAttributes.Count, messageAttributes.Count);
foreach (var ma in publishRequest.MessageAttributes)
{
var name = ma.Key;
var value = ma.Value;
Assert.True(messageAttributes.PropertyNames.Contains(name, StringComparer.Ordinal));
var jsonAttribute = messageAttributes[name];
var jsonType = jsonAttribute["Type"].ToString();
var jsonValue = jsonAttribute["Value"].ToString();
Assert.NotNull(jsonType);
Assert.NotNull(jsonValue);
Assert.Equal(value.DataType, jsonType);
Assert.Equal(value.DataType != "Binary"
? value.StringValue
: Convert.ToBase64String(value.BinaryValue.ToArray()), jsonValue);
}
await sqsClient.DeleteMessageAsync(new DeleteMessageRequest
{
QueueUrl = queueUrl,
ReceiptHandle = messages[0].ReceiptHandle
});
// This will unsubscribe but leave the policy in place.
await Client.UnsubscribeAsync(new UnsubscribeRequest
{
SubscriptionArn = subscriptionArn
});
// Subscribe again to see if this affects the policy.
await Client.SubscribeQueueAsync(topicArn, sqsClient as ICoreAmazonSQS, queueUrl);
await Client.PublishAsync(new PublishRequest
{
TopicArn = topicArn,
Message = "Test Message again"
});
messages = (await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = queueUrl,
WaitTimeSeconds = 20
})).Messages;
Assert.Equal(1, messages.Count);
var response = WaitUtils.WaitForComplete(
() => {
return(sqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
AttributeNames = new List <string> {
"All"
},
QueueUrl = queueUrl
}).Result);
},
(r) =>
{
return(!string.IsNullOrEmpty(r.Policy));
});
var policy = Policy.FromJson(response.Policy);
Assert.Equal(1, policy.Statements.Count);
}
private void SubscriberThread()
{
if (sqlController.SettingRead(Settings.token) != "UNIT_TEST___________________L:32")
#region amazon
{
#region setup
isActive = true;
keepSubscribed = true;
string awsAccessKeyId = sqlController.SettingRead(Settings.awsAccessKeyId);
string awsSecretAccessKey = sqlController.SettingRead(Settings.awsSecretAccessKey);
string awsQueueUrl = sqlController.SettingRead(Settings.awsEndPoint) + sqlController.SettingRead(Settings.token);
var sqsClient = new AmazonSQSClient(awsAccessKeyId, awsSecretAccessKey, RegionEndpoint.EUCentral1);
DateTime lastExpection = DateTime.MinValue;
DateTime lastCheckAdd15s;
#endregion
while (keepSubscribed)
{
try
{
lastCheckAdd15s = DateTime.Now.AddSeconds(15);
var res = sqsClient.ReceiveMessageAsync(awsQueueUrl).Result;
if (res.Messages.Count > 0)
{
foreach (var message in res.Messages)
{
#region JSON -> var
var parsedData = JRaw.Parse(message.Body);
string notificationUId = parsedData["id"].ToString();
string microtingUId = parsedData["microting_uuid"].ToString();
string action = parsedData["text"].ToString();
#endregion
log.LogStandard(t.GetMethodName("Subscriber"), "Notification notificationUId : " + notificationUId + " microtingUId : " + microtingUId + " action : " + action);
switch (action)
{
case Constants.Notifications.Completed:
sqlController.NotificationCreate(notificationUId, microtingUId, Constants.Notifications.Completed);
bus.SendLocal(new EformCompleted(notificationUId, microtingUId));
break;
case Constants.Notifications.EformParsedByServer:
bus.SendLocal(new EformParsedByServer(notificationUId, microtingUId));
break;
case Constants.Notifications.EformParsingError:
bus.SendLocal(new EformParsingError(notificationUId, microtingUId));
break;
case Constants.Notifications.RetrievedForm:
sqlController.NotificationCreate(notificationUId, microtingUId, Constants.Notifications.RetrievedForm);
bus.SendLocal(new EformRetrieved(notificationUId, microtingUId));
break;
case Constants.Notifications.UnitActivate:
sqlController.NotificationCreate(notificationUId, microtingUId, Constants.Notifications.UnitActivate);
bus.SendLocal(new UnitActivated(notificationUId, microtingUId));
break;
case Constants.Notifications.SpeechToTextCompleted:
sqlController.NotificationCreate(notificationUId, microtingUId, Constants.Notifications.SpeechToTextCompleted);
bus.SendLocal(new TranscriptionCompleted(notificationUId, microtingUId));
break;
}
sqsClient.DeleteMessageAsync(awsQueueUrl, message.ReceiptHandle);
}
}
else
{
while (lastCheckAdd15s > DateTime.Now)
{
Thread.Sleep(500);
}
}
}
catch (Exception ex)
{
//Log expection
log.LogWarning(t.GetMethodName("Subscriber"), t.PrintException(t.GetMethodName("Subscriber") + " failed", ex));
if (DateTime.Compare(lastExpection.AddMinutes(5), DateTime.Now) > 0)
{
keepSubscribed = false;
log.LogException(t.GetMethodName("Subscriber"), "failed, twice in the last 5 minuts", ex, true);
}
lastExpection = DateTime.Now;
}
}
//EventMsgClient("Subscriber closed", null);
keepSubscribed = false;
isActive = false;
}
#endregion
else
#region unit test
{
log.LogStandard(t.GetMethodName("Subscriber"), "Subscriber faked");
isActive = true;
keepSubscribed = true;
while (keepSubscribed)
{
Thread.Sleep(100);
}
keepSubscribed = false;
isActive = false;
}
#endregion
}