public static async Task ProcessMailAsync([QueueTrigger(QueueStore.NEW_MAIL)] CloudQueueMessage message, [Queue(QueueStore.NEW_MAIL)] CloudQueue nextQueue)
{
Console.WriteLine($"Started {QueueStore.NEW_MAIL}: {DateTime.UtcNow.ToLocalString()}");
var msg = message.GetBaseQueueMessage();
try
{
await _emailProvider.SendEmailAsync(msg.DeserializeData <MailQueueMessage>());
}
catch (Exception e)
{
Console.WriteLine($"Exception {QueueStore.NEW_MAIL}: {DateTime.UtcNow.ToLocalString()}, Retry {msg.RetryNext}: {e.Message}");
if (!msg.RetryNext)
{
// if a message is catched here then we inform the admin.
await _exceptionHandler.ProcessExceptionAsync(QueueStore.NEW_MAIL, e);
throw;
}
await nextQueue.AddMessageAsync(
CloudQueueMessage.CreateCloudQueueMessageFromByteArray(
msg.NextMessage.AsBytes()
),
null,
TimeSpan.FromSeconds(msg.NextMessage.TriggerDelay),
new QueueRequestOptions(),
new OperationContext()
);
}
finally
{
Console.WriteLine($"Finished {QueueStore.NEW_MAIL}: {DateTime.UtcNow.ToLocalString()}");
}
}
public async Task Queue(string routingKey, object message)
{
CloudQueue azureQueue = _azureQueueClient.GetQueueReference(routingKey.ToLower());
await azureQueue.AddMessageAsync(
CloudQueueMessage.CreateCloudQueueMessageFromByteArray(_serializationService.SerializeToBytes(message))
);
}
public async Task Test_13_MaxMessageSize()
{
var maxMsgSize = (int)CloudQueueMessage.MaxMessageSize;
Check.That(maxMsgSize).IsEqualTo(64 * 1024);
var testQName = "test-q-max-message";
var theQueue = _devQueueClient.GetQueueReference(testQName);
await theQueue.CreateIfNotExistsAsync();
await theQueue.ClearAsync();
var rawData = new byte[48 * 1024]; // 48 KB
var binaryMsg = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(rawData);
await theQueue.AddMessageAsync(binaryMsg);
var data = string.Empty.PadLeft(48 * 1024, '*');
Check.That(Convert.ToBase64String(UTF8Encoding.UTF8.GetBytes(data)).Length).IsStrictlyLessThan(maxMsgSize);
var sMsg = new CloudQueueMessage(data);
await theQueue.AddMessageAsync(sMsg);
}
public void QueueTrigger_WithNonTextualByteArrayMessageUsingQueueTriggerBindingData_Throws()
{
try
{
// Arrange
var host = JobHostFactory.Create <QueueTriggerBindingDataProgram>(
CloudStorageAccount.DevelopmentStorageAccount);
MethodInfo methodInfo = typeof(QueueTriggerBindingDataProgram).GetMethod("OnQueue");
byte[] contents = new byte[] { 0x00, 0xFF }; // Not valid UTF-8
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(contents);
// Act & Assert
FunctionInvocationException exception = Assert.Throws <FunctionInvocationException>(
() => host.Call(methodInfo, new { message = message }));
// This exeption shape/message could be better, but it's meets a minimum acceptibility threshold.
Assert.Equal("Exception binding parameter 'queueTrigger'", exception.InnerException.Message);
Exception innerException = exception.InnerException.InnerException;
Assert.IsType <InvalidOperationException>(innerException);
Assert.Equal("Binding data does not contain expected value 'queueTrigger'.", innerException.Message);
}
finally
{
QueueTriggerBindingDataProgram.QueueTrigger = null;
}
}
public async Task QueueTrigger_WithNonTextualByteArrayMessageNotUsingQueueTriggerBindingData_DoesNotThrow()
{
try
{
// Arrange
var host = new HostBuilder()
.ConfigureDefaultTestHost <QueueTriggerBindingDataProgram>(c =>
{
c.AddAzureStorage();
})
.Build()
.GetJobHost();
using (host)
{
MethodInfo methodInfo = typeof(QueueTriggerBindingDataProgram).GetMethod(nameof(QueueTriggerBindingDataProgram.ProcessQueueAsBytes));
byte[] expectedBytes = new byte[] { 0x00, 0xFF }; // Not valid UTF-8
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(expectedBytes);
// Act
await host.CallAsync(methodInfo, new { message = message });
// Assert
Assert.Equal(QueueTriggerBindingDataProgram.Bytes, expectedBytes);
}
}
finally
{
QueueTriggerBindingDataProgram.QueueTrigger = null;
}
}
public async Task QueueTrigger_WithNonTextualByteArrayMessageUsingQueueTriggerBindingData_Throws()
{
try
{
// Arrange
var host = new HostBuilder()
.ConfigureDefaultTestHost <QueueTriggerBindingDataProgram>(c =>
{
c.AddAzureStorage();
})
.Build()
.GetJobHost();
using (host)
{
MethodInfo methodInfo = typeof(QueueTriggerBindingDataProgram).GetMethod(nameof(QueueTriggerBindingDataProgram.OnQueue));
byte[] contents = new byte[] { 0x00, 0xFF }; // Not valid UTF-8
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(contents);
// Act & Assert
FunctionInvocationException exception = await Assert.ThrowsAsync <FunctionInvocationException>(
() => host.CallAsync(methodInfo, new { message = message }));
// This exeption shape/message could be better, but it's meets a minimum acceptibility threshold.
Assert.Equal("Exception binding parameter 'queueTrigger'", exception.InnerException.Message);
Exception innerException = exception.InnerException.InnerException;
Assert.IsType <InvalidOperationException>(innerException);
Assert.Equal("Binding data does not contain expected value 'queueTrigger'.", innerException.Message);
}
}
finally
{
QueueTriggerBindingDataProgram.QueueTrigger = null;
}
}
public async Task AddMessageToQueueAsync(string queueName, Guid id)
{
using (var ms = new MemoryStream())
{
Serializer.Serialize(ms, new BaseQueueMessage(id));
await AddMessageToQueueAsync(queueName, CloudQueueMessage.CreateCloudQueueMessageFromByteArray(ms.ToArray()));
}
}
public async Task SendAsync(EmailQueueToken token, CancellationToken cancellationToken)
{
_logger.LogTrace("Sending new queue message {0}", token);
if (!_initialized)
{
await InitializeAsync(cancellationToken);
}
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(token.EncodeBytes());
await _queue.Value.AddMessageAsync(message, null, null, null, null, cancellationToken);
}
public async Task EnqueueAsync(string queueName, byte[] source, TimeSpan?ttl = null, TimeSpan?initialVisibilityDelay = null)
{
CloudQueue queue = GetQueue(queueName);
if (!await queue.ExistsAsync())
{
throw new InvalidOperationException("Cloud queue does not exist.");
}
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(source);
await queue.AddMessageAsync(message, ttl, initialVisibilityDelay, new QueueRequestOptions(), null);
}
public void AddMessage(CloudQueueMessage message)
{
// need to create a new instance so other queues handling this message aren't updated
DateTimeOffset now = DateTimeOffset.Now;
var newMessage = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(message.AsBytes);
newMessage.SetId(Guid.NewGuid().ToString());
newMessage.SetInsertionTime(now);
newMessage.SetExpirationTime(now.AddDays(7));
newMessage.SetNextVisibleTime(now);
_visibleMessages.Enqueue(newMessage);
}
public override async Task <string> EnqueueAsync(TData data)
{
EnsureArg.IsNotNull(data, nameof(data));
this.EnsureMetaData(data);
var correlationId = data.As <IHaveCorrelationId>()?.CorrelationId ?? IdGenerator.Instance.Next;
using (this.Logger.BeginScope(new Dictionary <string, object>
{
[LogPropertyKeys.CorrelationId] = correlationId,
}))
using (var scope = this.Options.Tracer?.BuildSpan($"enqueue {this.Options.QueueName}", LogKeys.Queueing, SpanKind.Producer).Activate(this.Logger))
{
await this.EnsureQueueAsync().AnyContext();
this.Logger.LogDebug($"{{LogKey:l}} queue item enqueue (queue={this.Options.QueueName}, type={this.GetType().PrettyName()})", LogKeys.Queueing);
Interlocked.Increment(ref this.enqueuedCount);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(this.Serializer.SerializeToBytes(data));
// TODO: store correlationid + traceid/spanid >> QUEUEITEM > data
//using (var item = new QueueItem<TData>(IdGenerator.Instance.Next, data, this))
//{
// item.Properties.Add("TraceId", scope?.Span?.TraceId);
// item.Properties.Add("SpanId", scope?.Span?.SpanId);
// var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(this.Serializer.SerializeToBytes(item));
// await this.queue.AddMessageAsync(message).AnyContext();
// this.Logger.LogJournal(LogKeys.Queueing, $"item enqueued (id={message.Id}, queue={this.Options.QueueName}, data={typeof(TData).PrettyName()})", LogPropertyKeys.TrackEnqueue);
// this.Logger.LogTrace(LogKeys.Queueing, message.Id, typeof(TData).PrettyName(), LogTraceNames.Queue);
// this.LastEnqueuedDate = DateTime.UtcNow;
// return message.Id;
//}
var policy = Policy.Handle <Exception>()
.WaitAndRetry(this.Options.Retries, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) =>
{
this.Logger.LogWarning(ex, "{LogKey:l} could not enqueue item: {MessageId} after {Timeout}s ({ExceptionMessage})", LogKeys.AppMessaging, message.Id, $"{time.TotalSeconds:n1}", ex.Message);
});
await policy.Execute(async() =>
{
await this.queue.AddMessageAsync(message, this.Options.Expiration, TimeSpan.Zero, new QueueRequestOptions(), new OperationContext()).AnyContext();
}).AnyContext();
this.Logger.LogJournal(LogKeys.Queueing, $"queue item enqueued: {typeof(TData).PrettyName()} (id={message.Id}, queue={this.Options.QueueName})", LogPropertyKeys.TrackEnqueue);
this.Logger.LogTrace(LogKeys.Queueing, message.Id, typeof(TData).PrettyName(), LogTraceNames.Queue);
this.LastEnqueuedDate = DateTime.UtcNow;
return(message.Id);
}
}
public async Task MoveToPoisonQueueAsync(AzureEmailQueueMessage message, CancellationToken cancellationToken)
{
_logger.LogTrace("Moving message {0} to poison queue", message.Token);
if (!_initialized)
{
await InitializeAsync(cancellationToken);
}
var cloudQueueMessage = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(message.Token.EncodeBytes());
await _queue.Value.DeleteMessageAsync(message.MessageId, message.PopReceipt);
await _poisonQueue.Value.AddMessageAsync(cloudQueueMessage, null, null, null, null, cancellationToken);
}
public void AddMessage(MutableStorageQueueMessage message)
{
// need to create a new instance so other queues handling this message aren't updated
DateTimeOffset now = DateTimeOffset.Now;
var newMessage = new FakeStorageQueueMessage(CloudQueueMessage.CreateCloudQueueMessageFromByteArray(message.AsBytes))
{
Id = Guid.NewGuid().ToString(),
InsertionTime = now,
ExpirationTime = now.AddDays(7),
NextVisibleTime = now
};
_visibleMessages.Enqueue(newMessage);
}
private static async Task TestBindToByteArray(byte[] expectedContent)
{
// Arrange
var account = CreateFakeStorageAccount();
CloudQueue queue = await CreateQueue(account, QueueName);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(expectedContent);
await queue.AddMessageAsync(message);
// Act
byte[] result = await RunTriggerAsync<byte[]>(account, typeof(BindToByteArrayProgram),
(s) => BindToByteArrayProgram.TaskSource = s);
// Assert
Assert.Equal(expectedContent, result);
}
private async Task PostAppointmentToQueueStorageAsync(AppointmentDto appointment)
{
m_Logger.Information($"{nameof(PostAppointmentToQueueStorageAsync)} Invoked");
try
{
CloudQueue queue = GetQueue();
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(
AppointmentDto.Serialize <AppointmentDto>(appointment));
await queue.AddMessageAsync(message).ConfigureAwait(false);
}
catch (Exception ex)
{
m_Logger.Error(ex, "Error caught posting appointment to queue storage.");
}
}
Task Send(MessageWrapper wrapper, CloudQueue sendQueue, TimeSpan timeToBeReceived)
{
CloudQueueMessage rawMessage;
using (var stream = new MemoryStream())
{
serializer.Serialize(wrapper, stream);
#if NET452
rawMessage = new CloudQueueMessage(stream.ToArray());
#else
rawMessage = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(stream.ToArray());
#endif
}
return(sendQueue.AddMessageAsync(rawMessage, timeToBeReceived, null, null, null));
}
public async Task QueueTrigger_IfMessageIsUtf8ByteArray_ProvidesQueueTriggerBindingData()
{
// Arrange
const string expectedQueueTrigger = "abc";
byte[] content = Encoding.UTF8.GetBytes(expectedQueueTrigger);
var account = CreateFakeStorageAccount();
var queue = await CreateQueue(account, QueueName);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(content);
await queue.AddMessageAsync(message);
// Act
string result = await RunTriggerAsync<string>(account, typeof(BindToQueueTriggerBindingDataProgram),
(s) => BindToQueueTriggerBindingDataProgram.TaskSource = s);
// Assert
Assert.Equal(expectedQueueTrigger, result);
}
protected override async Task <string> EnqueueImplAsync(T data)
{
if (!await OnEnqueuingAsync(data).AnyContext())
{
return(null);
}
Interlocked.Increment(ref _enqueuedCount);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(_serializer.SerializeToBytes(data));
await _queueReference.AddMessageAsync(message).AnyContext();
var entry = new QueueEntry <T>(message.Id, data, this, SystemClock.UtcNow, 0);
await OnEnqueuedAsync(entry).AnyContext();
return(message.Id);
}
public async Task QueueTriggerByteArray()
{
TestHelpers.ClearFunctionLogs("QueueTriggerByteArray");
// write a binary queue message
byte[] inputBytes = new byte[] { 1, 2, 3 };
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(inputBytes);
var queue = Fixture.QueueClient.GetQueueReference("test-input-byte");
await queue.CreateIfNotExistsAsync();
await queue.ClearAsync();
await queue.AddMessageAsync(message);
JObject testResult = await GetFunctionTestResult("QueueTriggerByteArray");
Assert.True((bool)testResult["isBuffer"]);
Assert.Equal(5, (int)testResult["length"]);
}
public async Task EnqueueAsync(CrawlingQueueItem crawlingQueueItem, CancellationTokenSource cts)
{
// 1. Asign an AsyncState object when creating a task, so that if remote action fails, we still have the object to insert into local queue
// 2. Use a timeout value for remote operation
crawlingQueueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.InProxyQueue);
await _azureCrawlingQueue.AddMessageAsync(
CloudQueueMessage.CreateCloudQueueMessageFromByteArray(crawlingQueueItem.ToByteArray()), //new CloudQueueMessage(crawlingQueueItem.ToByteArray()),
null,
null,
new QueueRequestOptions
{
},
new OperationContext
{
},
cts.Token
);
}
public async Task QueueTrigger_IfMessageIsNonUtf8ByteArray_DoesNotProvideQueueTriggerBindingData()
{
// Arrange
byte[] content = new byte[] { 0xFF, 0x00 }; // Not a valid UTF-8 byte sequence.
var account = CreateFakeStorageAccount();
var queue = await CreateQueue(account, QueueName);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(content);
await queue.AddMessageAsync(message);
// Act
Exception exception = await RunTriggerFailureAsync<string>(account, typeof(BindToQueueTriggerBindingDataProgram),
(s) => BindToQueueTriggerBindingDataProgram.TaskSource = s);
// Assert
Assert.IsType<InvalidOperationException>(exception);
Assert.Equal("Exception binding parameter 'queueTrigger'", exception.Message);
Exception innerException = exception.InnerException;
Assert.IsType<InvalidOperationException>(innerException);
Assert.Equal("Binding data does not contain expected value 'queueTrigger'.", innerException.Message);
}
private async Task SetupAsync(StorageAccount account, object contents)
{
CloudQueueMessage message;
if (contents is string str)
{
message = new CloudQueueMessage(str);
}
else if (contents is byte[] bytearray)
{
message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(bytearray);
}
else
{
throw new InvalidOperationException("bad test");
}
var queue = await CreateQueue(account, QueueName);
// message.InsertionTime is provided by FakeStorageAccount when the message is inserted.
await queue.AddMessageAsync(message);
}
public async Task QueueTrigger_IfBoundToStringAndMessageIsNotUtf8ByteArray_DoesNotBind()
{
// Arrange
byte[] content = new byte[] { 0xFF, 0x00 }; // Not a valid UTF-8 byte sequence.
var account = CreateFakeStorageAccount();
CloudQueue queue = await CreateQueue(account, QueueName);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(content);
await queue.AddMessageAsync(message);
// Act
Exception exception = await RunTriggerFailureAsync<string>(account, typeof(BindToStringProgram),
(s) => BindToStringProgram.TaskSource = s);
// Assert
Assert.IsType<InvalidOperationException>(exception);
Assert.Equal("Exception binding parameter 'message'", exception.Message);
Exception innerException = exception.InnerException;
Assert.IsType<DecoderFallbackException>(innerException);
Assert.Equal("Unable to translate bytes [FF] at index -1 from specified code page to Unicode.",
innerException.Message);
}
public async Task Test_13_MaxSize()
{
var maxMsgSize = (int)CloudQueueMessage.MaxMessageSize;
maxMsgSize.Should().Be(64 * 1024);
var testQname = "test-q-maxmsg";
var azQ = azClient.GetQueueReference(testQname);
await azQ.CreateIfNotExistsAsync();
var rawData = new byte[48 * 1024]; // 48kB
var bMsg = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(rawData);
await azQ.AddMessageAsync(bMsg);
var data = string.Empty.PadLeft(40 * 1024, '*');
Convert.ToBase64String(Encoding.UTF8.GetBytes(data)).Length.Should().BeLessThan(maxMsgSize);
var sMsg = new CloudQueueMessage(data);
await azQ.AddMessageAsync(sMsg);
}
public void QueueTrigger_WithNonTextualByteArrayMessageNotUsingQueueTriggerBindingData_DoesNotThrow()
{
try
{
// Arrange
var host = JobHostFactory.Create <QueueTriggerBindingDataProgram>(
CloudStorageAccount.DevelopmentStorageAccount);
MethodInfo methodInfo = typeof(QueueTriggerBindingDataProgram).GetMethod("ProcessQueueAsBytes");
byte[] expectedBytes = new byte[] { 0x00, 0xFF }; // Not valid UTF-8
CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(expectedBytes);
// Act
host.Call(methodInfo, new { message = message });
// Assert
Assert.Equal(QueueTriggerBindingDataProgram.Bytes, expectedBytes);
}
finally
{
QueueTriggerBindingDataProgram.QueueTrigger = null;
}
}
public async Task TableEntity_IfUpdatesPoco_PersistsUsingNativeTableTypes()
{
// Arrange
byte[] originalValue = new byte[] { 0x12, 0x34 };
byte[] expectedValue = new byte[] { 0x56, 0x78 };
StorageAccount account = CreateFakeStorageAccount();
CloudQueue triggerQueue = await account.CreateQueueAsync(TriggerQueueName);
await triggerQueue.AddMessageAsync(CloudQueueMessage.CreateCloudQueueMessageFromByteArray(expectedValue));
CloudTable table = await account.CreateTableAsync(TableName);
Dictionary <string, EntityProperty> originalProperties = new Dictionary <string, EntityProperty>
{
{ "Value", new EntityProperty(originalValue) }
};
table.Insert(new DynamicTableEntity(PartitionKey, RowKey, etag: null, properties: originalProperties));
// Act
RunTrigger(account, typeof(UpdatePocoWithByteArrayValueProgram));
// Assert
DynamicTableEntity entity = table.Retrieve <DynamicTableEntity>(PartitionKey, RowKey);
Assert.NotNull(entity);
IDictionary <string, EntityProperty> properties = entity.Properties;
Assert.NotNull(properties);
Assert.True(properties.ContainsKey("Value"));
EntityProperty property = properties["Value"];
Assert.NotNull(property);
Assert.Equal(EdmType.Binary, property.PropertyType);
Assert.Equal(expectedValue, property.BinaryValue);
}
public override async Task <string> EnqueueAsync(TData data)
{
EnsureArg.IsNotNull(data, nameof(data));
this.EnsureMetaData(data);
using (this.logger.BeginScope(new Dictionary <string, object>
{
[LogEventPropertyKeys.CorrelationId] = data.As <IHaveCorrelationId>()?.CorrelationId,
}))
{
await this.EnsureQueueAsync().AnyContext();
this.logger.LogDebug($"queue item enqueue (queue={this.options.Name})");
Interlocked.Increment(ref this.enqueuedCount);
var message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(this.serializer.SerializeToBytes(data));
// TODO: store correlationid?
await this.queue.AddMessageAsync(message).AnyContext();
this.logger.LogJournal(LogEventPropertyKeys.TrackEnqueue, $"{{LogKey:l}} item enqueued (id={message.Id}, queue={this.options.Name}, type={typeof(TData).PrettyName()})", args: new[] { LogEventKeys.Queueing });
this.LastEnqueuedDate = DateTime.UtcNow;
return(message.Id);
}
}
/// <inheritdoc />
public IStorageQueueMessage CreateMessage(byte[] content)
{
return(new StorageQueueMessage(CloudQueueMessage.CreateCloudQueueMessageFromByteArray(content)));
}
private void AddMessage(byte[] body)
{
queue.AddMessage(CloudQueueMessage.CreateCloudQueueMessageFromByteArray(body));
}
