public void FailJsonWithEmptyMessageParts()
{
string message = ValidMessage;
// testing date
message = message.Replace(@"""date"": ""2020-02-11T17:38:36.9218638Z""", @"""date"": """"");
var(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing trigram
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing trigram
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing message
message = message.Replace(@"""message"" : ""Life Test Requested""", @"""message"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing level
message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEBUG""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing level must be uppercase
message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEbUG""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing length (must be 3)
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""ABCD""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""AD""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
}
public static async Task Run([EventHubTrigger("%EMP_EVENT_HUB_NAME%", Connection = "EMP_EVENTHUB_CONNECTION_STRING")] EventData[] events, ILogger log, ExecutionContext executionContext)
{
var exceptions = new List <Exception>();
string infoMessage;
string errorMessage;
string jsonSettingsPath = executionContext.FunctionAppDirectory;
//App Settings from local.settings.json
string storageConnectionString = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_CONNECTION_STRING;
string teamsQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_TEAMS_QUEUE_NAME;
string errorQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_ERROR_QUEUE_NAME;
string trigramTableName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_TRIGRAM_TABLE_NAME;
string elasticSearchClusterURI = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_CLUSTER_URI;
string elasticSearchAPIId = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_ID;
string elasticSearchAPIKey = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_KEY;
foreach (EventData eventData in events)
{
JsonLogEntry logEntry = null;
string webhookUrl = null;
bool validTrigram = false;
string messageBody = null;
try
{
messageBody = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
log?.LogInformation($"FROM EVENTHUB {messageBody}");
if (string.IsNullOrEmpty(messageBody))
{
infoMessage = $"Task Run: Event is null";
log?.LogInformation(infoMessage);
}
else
{
// Validate semantically the json schema
logEntry = JsonLogEntry.DeserializeJsonLogEntry(messageBody, log);
var(validJson, errorValidation) = JsonLogEntry.ValidateJsonLogEntry(messageBody, log);
//Validate Trigram - compare entry from the json log with the trigram azure table storage
(validTrigram, webhookUrl) = Trigram.ValidateTrigram(storageConnectionString, logEntry, trigramTableName, log);
if (validJson)
{
infoMessage = $"Task Run: Json Schema valid: {messageBody}";
log?.LogInformation(infoMessage);
if (validTrigram)
{
//Insert into ElasticSearch Cluster
var elasticLowLevelClient = emp_elastic_operations.ElasticConnect(elasticSearchClusterURI, elasticSearchAPIId, elasticSearchAPIKey, log);
await emp_elastic_operations.ElasticPutAsync(elasticLowLevelClient, logEntry, log);
}
else
{
// If not successful send to the Azure Storage Teams queue
log?.LogInformation($"Task Run: Application Trigram NOT valid: {messageBody}");
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = $"Invalid trigram", LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
}
}
else
{
// If not valid send to the Azure Storage Teams queue
infoMessage = $"Task Run: Json Schema NOT valid: {messageBody}";
log?.LogInformation(infoMessage);
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = errorValidation, LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
}
await Task.Yield();
}
}
catch (Exception ex)
{
if (logEntry != null)
{
// send to the Azure Storage teams queue
log?.LogInformation($"Task Run: exception raised {ex}");
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = $"{ex}", LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
// send to the Azure Storage Error queue
log?.LogInformation($"Task Run: exception raised {ex}");
cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, errorQueueName, log);
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, messageBody, log);
}
}
}
// Once processing of the batch is complete, if any messages in the batch failed processing throw an exception so that there is a record of the failure.
if (exceptions.Count > 1)
{
throw new AggregateException(exceptions);
}
if (exceptions.Count == 1)
{
throw exceptions.Single();
}
}
public void FailJsonWithMinimalRequirements()
{
var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(NonValidMessage, null);
Assert.False(emp);
}
public void FailJsonWithUndeserializableMessage()
{
Assert.Throws <JsonReaderException>(() => JsonLogEntry.ValidateJsonLogEntry("Undeserializable message", null));
}
public void ValidateJsonWithMinimalRequirements()
{
var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(ValidMessage, null);
Assert.True(emp);
}