public async System.Threading.Tasks.Task <List <SensorDataModel> > GetSensorDataAsync() { List <SensorDataModel> sensorReadingsList = new List <SensorDataModel>(); List <Message> messages = (await sqs.ReceiveMessageAsync(new ReceiveMessageRequest(myQueueUrl) { WaitTimeSeconds = 3, MaxNumberOfMessages = 10 })).Messages; foreach (var message in messages) { try { var snsMessage = Amazon.SimpleNotificationService.Util.Message.ParseMessage(message.Body); JObject reading = JObject.Parse(snsMessage.MessageText); var sensorReading = new SensorDataModel( (string)reading["value"], (string)reading["eventId"], (string)reading["locationId"], (string)reading["timestamp"] ); sensorReadingsList.Add(sensorReading); } catch { continue; } } if (messages.Count > 0) { await sqs.DeleteMessageBatchAsync(myQueueUrl, messages.Select(x => new DeleteMessageBatchRequestEntry(x.MessageId, x.ReceiptHandle)).ToList()); } return(sensorReadingsList); }
public override async Task ConfirmMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken = default) { var request = new DeleteMessageBatchRequest(_queueUrl, messages.Select(m => new DeleteMessageBatchRequestEntry(m.Id, m.Properties[Converter.ReceiptHandlePropertyName])).ToList()); await _client.DeleteMessageBatchAsync(request, cancellationToken).ConfigureAwait(false); }
private static async Task <List <Message> > ReadAmazonSqsEvent(string queueUrl, int waitTimeoutSeconds) { using (AmazonSQSClient sqsClient = new AmazonSQSClient()) { ReceiveMessageResponse receiveMessageResponse = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest(queueUrl) { WaitTimeSeconds = waitTimeoutSeconds, MessageAttributeNames = new List <string> { "All" } }); List <Message> messages = receiveMessageResponse.Messages.Select(GetMessage).ToList(); if (receiveMessageResponse.Messages.Any()) { List <DeleteMessageBatchRequestEntry> deleteMessageBatchRequestEntries = receiveMessageResponse .Messages .Select(_ => new DeleteMessageBatchRequestEntry(_.MessageId, _.ReceiptHandle)).ToList(); DeleteMessageBatchRequest deleteMessageBatchRequest = new DeleteMessageBatchRequest(queueUrl, deleteMessageBatchRequestEntries); DeleteMessageBatchResponse response = await sqsClient.DeleteMessageBatchAsync(deleteMessageBatchRequest); } return(messages); } }
/// <summary> /// Deletes all messages from the input queue (which is done by receiving them in batches and deleting them, as long as /// it takes) /// </summary> public void Purge(string queueUrl) { m_log.Info("Purging queue {queueUrl}", queueUrl); try { // we purge the queue by receiving all messages as fast as we can... //the native purge function is not used because it is only allowed to use it // once every 60 s using (var client = new AmazonSQSClient(m_AmazonInternalSettings.AmazonCredentialsFactory.Create(), m_AmazonInternalSettings.AmazonSqsConfig)) { var stopwatch = Stopwatch.StartNew(); while (true) { var request = new ReceiveMessageRequest(queueUrl) { MaxNumberOfMessages = 10 }; var receiveTask = client.ReceiveMessageAsync(request); AsyncHelpers.RunSync(() => receiveTask); var response = receiveTask.Result; if (!response.Messages.Any()) { break; } var deleteTask = client.DeleteMessageBatchAsync(queueUrl, response.Messages.Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle)).ToList()); AsyncHelpers.RunSync(() => deleteTask); var deleteResponse = deleteTask.Result; if (deleteResponse.Failed.Any()) { var errors = string.Join(Environment.NewLine, deleteResponse.Failed.Select(f => $"{f.Message} ({f.Id})")); throw new RebusApplicationException($@"Error {deleteResponse.HttpStatusCode} while purging: {errors}"); } } m_log.Info("Purging {queueUrl} took {elapsedSeconds} s", queueUrl, stopwatch.Elapsed.TotalSeconds); } } catch (AmazonSQSException exception) when(exception.StatusCode == HttpStatusCode.BadRequest) { if (exception.Message.Contains("queue does not exist")) { return; } throw; } catch (Exception exception) { throw new RebusApplicationException(exception, $"Error while purging {queueUrl}"); } }
private static async Task RunLambda() { Environment.SetEnvironmentVariable("RemainingTimeThresholdSeconds", "10"); Environment.SetEnvironmentVariable("TimeoutSqsSeconds", "10"); AmazonSQSClient client = new AmazonSQSClient(new EnvironmentVariablesAWSCredentials()); LambdaEntryPoint pollerLambdaEntryPoint = new LambdaEntryPoint(); string queueUrl = Environment.GetEnvironmentVariable("SqsQueueUrl"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(queueUrl) { MaxNumberOfMessages = 1, WaitTimeSeconds = 20, MessageAttributeNames = new List <string> { "All" }, AttributeNames = new List <string> { "All" } }; while (true) { Console.WriteLine($"Polling {queueUrl} for messages..."); ReceiveMessageResponse receiveMessageResponse = await client.ReceiveMessageAsync(receiveMessageRequest); Console.WriteLine($"Received {receiveMessageResponse.Messages.Count} messages from {queueUrl}."); if (receiveMessageResponse.Messages.Any()) { try { Console.WriteLine("Running Lambda..."); SQSEvent sqsEvent = receiveMessageResponse.Messages.ToSqsEvent(); await pollerLambdaEntryPoint.FunctionHandler(sqsEvent, LambdaContext.NonExpiringLambda); Console.WriteLine("Lambda completed"); Console.WriteLine("Deleting messages..."); await client.DeleteMessageBatchAsync(queueUrl, sqsEvent.Records.Select(_ => new DeleteMessageBatchRequestEntry { Id = _.MessageId, ReceiptHandle = _.ReceiptHandle }).ToList()); Console.WriteLine("Deleted messages."); } catch (Exception e) { Console.WriteLine($"An error occured running lambda {e.Message} {Environment.NewLine} {e.StackTrace}"); } } } }
public static async Task Main(string[] args) { ///External Credentials AWSCredentials credentials = GetGasmonCredentials(); AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, RegionEndpoint.EUWest1); Dictionary <string, JsonFile> locationsById = await GetJson(amazonS3Client); var sns = new AmazonSimpleNotificationServiceClient(credentials, RegionEndpoint.EUWest1); var sqs = new AmazonSQSClient(credentials, RegionEndpoint.EUWest1); var myTopicArn = "arn:aws:sns:eu-west-1:552908040772:EventProcessing-UCAS2-snsTopicSensorDataPart2-6SW9NJKW1LZ9"; var myQueueName = "Joana-GasMon-" + Guid.NewGuid(); var myQueueUrl = (await sqs.CreateQueueAsync(myQueueName)).QueueUrl; await sns.SubscribeQueueAsync(myTopicArn, sqs, myQueueUrl); DateTime initTime = DateTime.Now; DateTime finalTime = initTime.AddMinutes(6); Console.WriteLine("The Initial time is: {0}", initTime); var fileName = ""; List <Amazon.SQS.Model.Message> sqsmessages; for (int i = 0; i < 6; i++) { Dictionary <string, MessageMsg> dictionary = new Dictionary <string, MessageMsg>(); DateTime loopTime = initTime.AddMinutes(i + 1); do { sqsmessages = (await sqs.ReceiveMessageAsync(new ReceiveMessageRequest(myQueueUrl) { WaitTimeSeconds = 1 })).Messages; GetMessages(locationsById, dictionary, initTime, finalTime, sqsmessages); var receipHandle = sqsmessages.Select(r => new DeleteMessageBatchRequestEntry(r.MessageId, r.ReceiptHandle)); await sqs.DeleteMessageBatchAsync(myQueueUrl, receipHandle.ToList()); }while (DateTime.Now < loopTime); fileName = WriteFile(dictionary, i, fileName, initTime); } Console.WriteLine("The Final time is: {0}", finalTime); Console.ReadKey(); }
private System.Collections.IEnumerator CheckNewSQSMessages(float waitTime) { while (true) { yield return(new WaitForSeconds(waitTime)); if (!string.IsNullOrEmpty(IncomingQueueUrl)) { mSQSClient.ReceiveMessageAsync(IncomingQueueUrl, (result) => { if (result.Exception == null) { var messages = result.Response.Messages; Debug.Log("Got " + messages.Count + " messages"); List <SQSMessage> newMessages = new List <SQSMessage>(); List <DeleteMessageBatchRequestEntry> deleteReq = new List <DeleteMessageBatchRequestEntry>(); messages.ForEach(m => { var msg = m.Body.FromJson <SQSMessage>(); msg.id = m.MessageId; newMessages.Add(msg); deleteReq.Add(new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle)); }); if (newMessages.Count > 0) { lock (mMessageQueue) { mMessageQueue.AddRange(newMessages); } mSQSClient.DeleteMessageBatchAsync(IncomingQueueUrl, deleteReq, (delResult) => { if (delResult.Exception != null) { Debug.LogException(delResult.Exception); } }); } } else { Debug.LogException(result.Exception); } }); } else { Debug.Log(@"Queue Url is empty, make sure that the queue is created first"); } } }
/// <inheritdoc/> public async Task ConfirmAsync(IEnumerable <Message> messages, CancellationToken cancellationToken = default) { _ = messages ?? throw new ArgumentNullException(nameof(messages)); var response = await _client.DeleteMessageBatchAsync(new DeleteMessageBatchRequest { QueueUrl = await GetQueueUrl().ConfigureAwait(false), Entries = messages.Select(m => new DeleteMessageBatchRequestEntry { Id = m.Id, ReceiptHandle = (string)m.SystemProperties[ReceiptHandleProperty] }).ToList() }, cancellationToken); }
private async Task RemoveProcessedMessagesFromQueue(List <DeleteMessageBatchRequestEntry> processedMessages) { if (processedMessages.Count > 0) { var client = new AmazonSQSClient(); var delRequest = new DeleteMessageBatchRequest { Entries = processedMessages, QueueUrl = Environment.GetEnvironmentVariable("LoansEventsQueue") }; await client.DeleteMessageBatchAsync(delRequest); } }
public async Task <HttpStatusCode> DeleteMessages(IEnumerable <string> receiptHandles, string url = null) { if (url == null) { url = awsSQSOptions.Url; } var sendMessageResponse = await amazonSQSClient.DeleteMessageBatchAsync(url, receiptHandles.Select((handle, index) => new DeleteMessageBatchRequestEntry() { Id = index.ToString(), ReceiptHandle = handle }).ToList()); return(sendMessageResponse.HttpStatusCode); }
public async Task <IEnumerable <IAwsArgs> > Dequeue() { //Retrieve message var recieveRequest = new ReceiveMessageRequest(_queueUrl); var response = await _client.ReceiveMessageAsync(recieveRequest); //Initialize return collections var returnList = response.Messages.Select(msg => JsonConvert.DeserializeObject <AwsEventArgs>(msg.Body)).Cast <IAwsArgs>().ToList(); //Delete messages var deleteList = response.Messages.Select(msg => new DeleteMessageBatchRequestEntry(msg.MessageId, msg.ReceiptHandle)).ToList(); var result = await _client.DeleteMessageBatchAsync(_queueUrl, deleteList); //Return messages return(returnList); }
public void RemoveObjectsFromQueue <T>(List <T> objects) where T : Models.Dynamo.ReceiptBase { if (objects != null && objects.Count > 0) { Console.WriteLine($"Removing list {JsonConvert.SerializeObject(objects)}"); using (var sqsClient = new AmazonSQSClient()) { var result = sqsClient.DeleteMessageBatchAsync(new DeleteMessageBatchRequest { Entries = objects.Select(x => new DeleteMessageBatchRequestEntry { Id = Guid.NewGuid().ToString(), ReceiptHandle = x.ReceiptHandle }).ToList(), QueueUrl = Environment.GetEnvironmentVariable("Queue") }).Result.Successful; } } }
private static async Task PurgeAmazonSqsQueue(string queueUrl) { using (AmazonSQSClient sqsClient = new AmazonSQSClient()) { ReceiveMessageResponse receiveMessageResponse; do { receiveMessageResponse = await sqsClient.ReceiveMessageAsync( new ReceiveMessageRequest(queueUrl) { MaxNumberOfMessages = 10 }); if (receiveMessageResponse.Messages.Any()) { DeleteMessageBatchResponse deleteMessageResponse = await sqsClient.DeleteMessageBatchAsync(new DeleteMessageBatchRequest(queueUrl, receiveMessageResponse.Messages.Select(_ => new DeleteMessageBatchRequestEntry(_.MessageId, _.ReceiptHandle)).ToList())); } } while (receiveMessageResponse.Messages.Any()); } }
private static async Task ProcessMessages(CommandOption queueUrl, TlsRptSqsSchedulerLambdaEntryPoint entryPoint, ReceiveMessageRequest receiveMessageRequest) { AmazonSQSClient client = new AmazonSQSClient(new EnvironmentVariablesAWSCredentials()); Console.WriteLine($@"Polling {queueUrl.Value()} for messages..."); ReceiveMessageResponse receiveMessageResponse = await client.ReceiveMessageAsync(receiveMessageRequest); Console.WriteLine($@"Received {receiveMessageResponse.Messages.Count} messages from {queueUrl.Value()}."); if (receiveMessageResponse.Messages.Any()) { try { Console.WriteLine(@"Running Lambda..."); SQSEvent sqsEvent = receiveMessageResponse.Messages.ToSqsEvent(); await entryPoint.FunctionHandler(sqsEvent, LambdaContext.NonExpiringLambda); Console.WriteLine(@"Lambda completed."); Console.WriteLine(@"Deleting messages..."); await client.DeleteMessageBatchAsync(queueUrl.Value(), sqsEvent.Records.Select(_ => new DeleteMessageBatchRequestEntry { Id = _.MessageId, ReceiptHandle = _.ReceiptHandle }).ToList()); Console.WriteLine(@"Messages deleted."); } catch (Exception e) { Console.WriteLine($@"An error occured running lambda {e.Message} {Environment.NewLine} {e.StackTrace}"); } } }
public async Task <IEnumerable <String> > ReceiveAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { var receiveResponse = await amazonSqsClient.ReceiveMessageAsync(new ReceiveMessageRequest() { MaxNumberOfMessages = 1, QueueUrl = queueUrl, WaitTimeSeconds = 20, VisibilityTimeout = 60 }, cancellationToken); if (receiveResponse.Messages.Any()) { var deleteRequestEntries = receiveResponse .Messages .Select(message => new DeleteMessageBatchRequestEntry() { Id = message.MessageId, ReceiptHandle = message.ReceiptHandle }) .ToList(); await amazonSqsClient.DeleteMessageBatchAsync(new DeleteMessageBatchRequest() { Entries = deleteRequestEntries, QueueUrl = queueUrl }); return(receiveResponse.Messages.Select(message => { var amazonSqsMessage = jsonParser.Parse <AmazonSqsMessage>(message.Body); return amazonSqsMessage.Message; })); } } return(Enumerable.Empty <String>()); }
/// <summary> /// /// <para>CustomSubscribe:</para> /// /// <para>Subscribes to given custom topic</para> /// /// <para>Check <seealso cref="IBPubSubServiceInterface.CustomSubscribe"/> for detailed documentation</para> /// /// </summary> public bool CustomSubscribe(string _CustomTopic, Action <string, string> _OnMessage, Action <string> _ErrorMessageAction = null, bool _SubscribeSingleMessage = false) { if (_CustomTopic != null && _CustomTopic.Length > 0 && _OnMessage != null && BUtility.CalculateStringMD5(_CustomTopic, out string TopicMD5, _ErrorMessageAction)) { if (EnsureQueueExists(TopicMD5, out string QueueUrl, _ErrorMessageAction)) { var SubscriptionCancellationVar = new BValue <bool>(false, EBProducerStatus.MultipleProducer); var SubscriptionThread = new Thread(() => { Thread.CurrentThread.IsBackground = true; while (!SubscriptionCancellationVar.Get()) { ReceiveMessageResponse Response; try { using (var ReceiveMessageTask = SQSClient.ReceiveMessageAsync(QueueUrl)) { ReceiveMessageTask.Wait(); Response = ReceiveMessageTask.Result; } } catch (Exception e) { Response = null; _ErrorMessageAction?.Invoke("BPubSubServiceAWS->CustomSubscribe: " + e.Message + ", Trace: " + e.StackTrace); if (e.InnerException != null && e.InnerException != e) { _ErrorMessageAction?.Invoke("BPubSubServiceAWS->CustomSubscribe->Inner: " + e.InnerException.Message + ", Trace: " + e.InnerException.StackTrace); } } if (Response == null || Response.Messages == null || Response.Messages.Count == 0) { Thread.Sleep(1000); continue; } var AckDictionary = new Dictionary <string, string>(); foreach (var MessageContainer in Response.Messages) { if (MessageContainer != null) { if (!AckDictionary.ContainsKey(MessageContainer.MessageId)) { AckDictionary.Add(MessageContainer.MessageId, MessageContainer.ReceiptHandle); } string Data = MessageContainer.Body; if (UniqueMessageDeliveryEnsurer != null) { UniqueMessageDeliveryEnsurer.Subscribe_ClearAndExtractTimestampFromMessage(ref Data, out string TimestampHash); if (UniqueMessageDeliveryEnsurer.Subscription_EnsureUniqueDelivery(_CustomTopic, TimestampHash, _ErrorMessageAction)) { _OnMessage?.Invoke(_CustomTopic, Data); } } else { _OnMessage?.Invoke(_CustomTopic, Data); } } } var AckArray = new List <DeleteMessageBatchRequestEntry>(); foreach (var Current in AckDictionary) { AckArray.Add(new DeleteMessageBatchRequestEntry(Current.Key, Current.Value)); } try { using (var DeleteMessageBatchTask = SQSClient.DeleteMessageBatchAsync(QueueUrl, AckArray)) { DeleteMessageBatchTask.Wait(); } } catch (Exception e) { _ErrorMessageAction?.Invoke("BPubSubServiceAWS->CustomSubscribe: " + e.Message + ", Trace: " + e.StackTrace); if (e.InnerException != null && e.InnerException != e) { _ErrorMessageAction?.Invoke("BPubSubServiceAWS->CustomSubscribe->Inner: " + e.InnerException.Message + ", Trace: " + e.InnerException.StackTrace); } } } }); SubscriptionThread.Start(); lock (SubscriberThreadsDictionaryLock) { SubscriberThreadsDictionary.Add(_CustomTopic, new BTuple <Thread, BValue <bool> >(SubscriptionThread, SubscriptionCancellationVar)); } return(true); } } return(false); }
/// <summary> /// Deletes all messages from the input queue (which is done by receiving them in batches and deleting them, as long as it takes) /// </summary> public void Purge() { if (Address == null) { return; } _log.Info("Purging {0} (by receiving all messages from the queue)", Address); try { using (var client = new AmazonSQSClient(_credentials, _amazonSqsConfig)) { var stopwatch = Stopwatch.StartNew(); while (true) { var receiveTask = client.ReceiveMessageAsync(new ReceiveMessageRequest(_queueUrl) { MaxNumberOfMessages = 10 }); AsyncHelpers.RunSync(() => receiveTask); var response = receiveTask.Result; if (!response.Messages.Any()) { break; } var deleteTask = client.DeleteMessageBatchAsync(_queueUrl, response.Messages .Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle)) .ToList()); AsyncHelpers.RunSync(() => deleteTask); var deleteResponse = deleteTask.Result; if (deleteResponse.Failed.Any()) { var errors = string.Join(Environment.NewLine, deleteResponse.Failed.Select(f => $"{f.Message} ({f.Id})")); throw new RebusApplicationException( $@"Error {deleteResponse.HttpStatusCode} while purging: {errors}"); } } _log.Info($"Purging {Address} took {stopwatch.Elapsed.TotalSeconds:0.0} s"); } } catch (AmazonSQSException exception) when(exception.StatusCode == HttpStatusCode.BadRequest) { if (exception.Message.Contains("queue does not exist")) { return; } throw; } catch (Exception exception) { throw new RebusApplicationException(exception, $"Error while purging {Address}"); } }
private async Task PollQueue() { _logger.LogDebug("PollQueue() initiated."); while (true) { if (_cancellationToken.IsCancellationRequested) { break; } try { // Generic retry policy for out out proc stuff PolicyResult <ReceiveMessageResponse> policyResult = await Policy .Handle <Amazon.Runtime.AmazonServiceException>() //.OrResult<ReceiveMessageResponse>(r => r.HttpStatusCode != System.Net.HttpStatusCode.OK) .WaitAndRetryForeverAsync((x) => TimeSpan.FromSeconds(5), (e, t) => _logger.LogWarning(e, "Retrying SQS poll...")) .ExecuteAndCaptureAsync <ReceiveMessageResponse>(async() => { ReceiveMessageRequest request = GenerateQueueReceiveRequest(); _logger.LogDebug("Polling messages from SQS..."); var response = await _sqsClient.ReceiveMessageAsync(request, _cancellationToken); _logger.LogDebug("Polly execute returned the ReceiveMessageRequest. Status: {status}", response.HttpStatusCode); return(response); }); //will default(TResult) if policy failed if (policyResult.Outcome != OutcomeType.Successful) { _logger.LogError("Policy did not complete successfully for this poll. Policy Outcome: {policyOutcome}.", policyResult.Outcome); continue; } var receiveMessageResponse = policyResult.Result; if (receiveMessageResponse.HttpStatusCode == System.Net.HttpStatusCode.OK) { _logger.LogInformation("Message polling response came back with {numMessages} messages in queue.", receiveMessageResponse.Messages.Count); if (receiveMessageResponse.Messages.Count == 0) { continue; } // kick off each piece of work into a background task and only await the whole batch Task <Message>[] bgWorkTasks = StartMessageBackgroundTasks(receiveMessageResponse); await Task.WhenAll(bgWorkTasks); List <Task <Message> > successTasks = bgWorkTasks.Where(x => x.IsCompletedSuccessfully).ToList(); List <Task <Message> > failTasks = bgWorkTasks.Where(x => !x.IsCompletedSuccessfully).ToList(); _logger.LogInformation("{numSuccessTasks} tasks marked as completing successfully, and {numFailTasks} marked as not completing successfully.", successTasks.Count, failTasks.Count); // todo: what to do here? failTasks.ForEach(x => x.ContinueWith(y => _logger.LogError("Message failures...watdo???"))); // Batch delete the successful pieces, let go of the fails DeleteMessageBatchRequest batchDelete = GetBatchMessageRequest(successTasks.Select(x => x.Result).ToList()); var batchDeleteResponse = await _sqsClient.DeleteMessageBatchAsync(batchDelete, _cancellationToken); _logger.LogDebug("Message delete response: {statusCode} {@deleteResponseMetadata}", receiveMessageResponse.HttpStatusCode, batchDeleteResponse.ResponseMetadata); } else { _logger.LogError("Http status code on ReceiveMessageResponse did not "); } } catch (Exception e) { _logger.LogError(e, "Error on requesting messages from Queue."); } } }
public virtual async Task <DeleteMessageBatchResponse> DeleteMessageBatchAsync(DeleteMessageBatchRequest deleteMessageBatchRequest) { var deleteMessageResponse = await _amazonSqsClient.DeleteMessageBatchAsync(deleteMessageBatchRequest); return(deleteMessageResponse); }