/// <summary>
/// Function callback to post a message in Teams.
/// TODO: work on the "message" parameter, so far just a simple text mokup
/// </summary>
/// <remarks>
/// The object posted should be like this:
/// {
/// "message":"error test with objects",
/// "emp_json_log_entry": {
/// "trigram": "abc",
/// "application": "Axaim.Membership.Admin",
/// "layer": "lifetest",
/// "level": "DEBUG",
/// "date": "Z2020etcetc",
/// "message": "the original message",
/// "WebhookURL": ""
/// }
/// }
/// TODO: adjust this based on the real emp_json_log_entry class
/// </remarks>
/// <param name="req">The http request</param>
/// <param name="log">Logger to log</param>
/// <returns>a result action wit OK if all goes right or BadRequestObjectResult in case of issue</returns>
public static async Task <IActionResult> RunFromDataString(string requestBody, ILogger log)
{
// Get the message from the body
string message;
dynamic data;
try
{
data = JsonConvert.DeserializeObject(requestBody);
message = data?.message;
}
catch (JsonReaderException ex)
{
log?.LogError(ex, "Error deserializing the message");
return((ActionResult) new BadRequestObjectResult($"Error: {ex}"));
}
log?.LogInformation("Processing a request to post an error on a Teams channel");
if (message == null)
{
log?.LogError("No error message provided");
return(new BadRequestObjectResult("Please pass a name on the query string or in the request body"));
}
// Get the emp_json_log_entry from the body
JsonLogEntry logEntry = new JsonLogEntry();
logEntry.Trigram = data?.emp_log_entry?.trigram;
logEntry.Level = data?.emp_log_entry?.level;
logEntry.Date = data?.emp_log_entry?.date;
logEntry.Message = data?.emp_log_entry?.message;
string webhookURL = data?.WebhookURL;
return(await PostOnTeams(message, webhookURL, logEntry, log));
}
public void ValidVariousDates()
{
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9218638Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.92Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.921Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9218Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.92186Z"));
Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.921863Z"));
Assert.False(JsonLogEntry.IsValidIso8601Date(null));
Assert.False(JsonLogEntry.IsValidIso8601Date(string.Empty));
Assert.False(JsonLogEntry.IsValidIso8601Date("Non date format"));
Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11"));
Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11T11:15"));
Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11T11:15Z"));
}
protected System.Collections.Generic.IEnumerable <JsonLogEntry> GetEntries()
{
Newtonsoft.Json.JsonSerializer oJsonSerializer = new Newtonsoft.Json.JsonSerializer();
foreach (System.Tuple <string, System.IO.Stream> st in StreamFactory.GetStreams(From, this.Recurse))
{
this.StreamLineNumber = 0;
this.CurrentStreamName = st.Item1;
this.RecordIndex = 0;
using (AdvancedTextReader srdr = new AdvancedTextReader(st.Item2, this.TextReaderOptions))
{
using (var m_JsonReader = new Newtonsoft.Json.JsonTextReader(srdr)
{
SupportMultipleContent = true
})
{
bool bStartRead = string.IsNullOrEmpty(JsonPropertyToRead);
while (m_JsonReader.Read())
{
if (!bStartRead && !(m_JsonReader.TokenType == JsonToken.PropertyName && m_JsonReader.Value == JsonPropertyToRead))
{
continue;
}
else
{
bStartRead = true;
}
if (m_JsonReader.TokenType == JsonToken.StartObject)
{
// Additional fields
this.StreamLineNumber = m_JsonReader.LineNumber;
this.RecordIndex += 1;
// Deserialize entries and yield
JsonLogEntry le = oJsonSerializer.Deserialize <JsonLogEntry>(m_JsonReader);
yield return(le);
}
}
}
}
}
}
protected override void InitValueExtractor()
{
// Init value extractor from first record
ValueExtractor = new DictionaryExtractor(new string[] { }); // init empty first if can't init later
Newtonsoft.Json.JsonSerializer oJsonSerializer = new Newtonsoft.Json.JsonSerializer();
foreach (var st in StreamFactory.GetStreams(this.From, this.Recurse)) // returns touplse (stream name as string, Stream object implementaion)
{
using (AdvancedTextReader srdr = new AdvancedTextReader(st.Item2, this.TextReaderOptions))
{
using (var m_JsonReader = new Newtonsoft.Json.JsonTextReader(srdr)
{
SupportMultipleContent = true
})
{
bool bStartRead = string.IsNullOrEmpty(JsonPropertyToRead);
while (m_JsonReader.Read())
{
if (!bStartRead && !(m_JsonReader.TokenType == JsonToken.PropertyName && m_JsonReader.Value == JsonPropertyToRead))
{
continue;
}
else
{
bStartRead = true;
}
if (m_JsonReader.TokenType == JsonToken.StartObject)
{
// Deserialize
JsonLogEntry logentry = oJsonSerializer.Deserialize <JsonLogEntry>(m_JsonReader);
// Set the value extractor from the first JSON record and exit
ValueExtractor = new DictionaryExtractor(logentry);
break;
}
}
}
}
}
}
/// <summary>
/// Create a message card ready to publish in Teams
/// TODO: rather than string, need to create a class or have multiple parameters to the message to post
/// </summary>
/// <param name="message">The message to post see TODO</param>
/// <returns>A message card ready to post</returns>
public static MessageCard CreateMessageCard(string message, JsonLogEntry logEntry)
{
MessageCard card = new MessageCard();
card.Title = "Log entry error";
card.Text = $"Error: {message}";
card.Sections = new List <Section>
{
new Section
{
ActivityTitle = logEntry?.Trigram,
ActivitySubtitle = DateTime.Now.ToString(),
Facts = new List <Fact>
{
new Fact {
Name = $"{nameof(logEntry.Date)}", Value = logEntry?.Date
},
new Fact {
Name = $"{nameof(logEntry.Level)}", Value = logEntry?.Level
}
},
Text = $"Original message: {logEntry?.Message}"
}
};
card.Actions = new List <IAction>
{
new OpenUriAction
{
Type = ActionType.OpenUri,
Name = "View on site",
Targets = new List <Target> {
new Target {
OS = TargetOs.Default, Uri = string.Format(Environment.GetEnvironmentVariable(DefaultKibanaUrl), logEntry?.Trigram)
}
}
}
};
return(card);
}
public void FailJsonWithEmptyMessageParts()
{
string message = ValidMessage;
// testing date
message = message.Replace(@"""date"": ""2020-02-11T17:38:36.9218638Z""", @"""date"": """"");
var(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing trigram
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing trigram
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing message
message = message.Replace(@"""message"" : ""Life Test Requested""", @"""message"" : """"");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing level
message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEBUG""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing level must be uppercase
message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEbUG""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
// testing length (must be 3)
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""ABCD""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""AD""");
(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null);
Assert.False(emp);
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
if (formatter is null)
{
throw new ArgumentNullException(nameof(formatter));
}
var message = new JsonLogEntry
{
Timestamp = DateTimeOffset.UtcNow,
LogLevel = logLevel.ToString(),
Category = _categoryName,
Exception = exception?.ToString(),
Message = formatter(state, exception),
};
// Append the data of all BeginScope and LogXXX parameters to the message dictionary
// REMOVED: AppendScope(message.Scope, state); - This line in original JsonLogger from Microsoft it's adding duplicated message details
AppendScope(message.Scope);
_writer.WriteLine(JsonConvert.SerializeObject(message, Formatting.None));
}
/// <summary>
/// Post a message in Teams.
/// </summary>
/// <param name="message">A message</param>
/// <param name="webhookURL">The webhook URL</param>
/// <param name="logEntry">A log entry</param>
/// <param name="log">The logger</param>
/// <returns>>a result action wit OK if all goes right or BadRequestObjectResult in case of issue</returns>
public static async Task <IActionResult> PostOnTeams(string message, string webhookURL, JsonLogEntry logEntry, ILogger log)
{
string url = string.IsNullOrEmpty(webhookURL) ? Environment.GetEnvironmentVariable(DefaultWebhookUrlEnvironmenent) : webhookURL;
if (string.IsNullOrEmpty(url))
{
log?.LogError($"No default {DefaultWebhookUrlEnvironmenent} environment variable provided");
return(new BadRequestObjectResult($"A default WebhookURL needs to be provided"));
}
MessageCard card;
// Prepare the message
if (logEntry != null)
{
card = CreateMessageCard(message, logEntry);
}
else
{
card = CreateBasicMessage(message);
}
var json = JsonConvert.SerializeObject(card);
return(await PostOnTeamsMessage(json, url, log));
}
public static async Task Run([EventHubTrigger("%EMP_EVENT_HUB_NAME%", Connection = "EMP_EVENTHUB_CONNECTION_STRING")] EventData[] events, ILogger log, ExecutionContext executionContext)
{
var exceptions = new List <Exception>();
string infoMessage;
string errorMessage;
string jsonSettingsPath = executionContext.FunctionAppDirectory;
//App Settings from local.settings.json
string storageConnectionString = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_CONNECTION_STRING;
string teamsQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_TEAMS_QUEUE_NAME;
string errorQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_ERROR_QUEUE_NAME;
string trigramTableName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_TRIGRAM_TABLE_NAME;
string elasticSearchClusterURI = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_CLUSTER_URI;
string elasticSearchAPIId = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_ID;
string elasticSearchAPIKey = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_KEY;
foreach (EventData eventData in events)
{
JsonLogEntry logEntry = null;
string webhookUrl = null;
bool validTrigram = false;
string messageBody = null;
try
{
messageBody = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
log?.LogInformation($"FROM EVENTHUB {messageBody}");
if (string.IsNullOrEmpty(messageBody))
{
infoMessage = $"Task Run: Event is null";
log?.LogInformation(infoMessage);
}
else
{
// Validate semantically the json schema
logEntry = JsonLogEntry.DeserializeJsonLogEntry(messageBody, log);
var(validJson, errorValidation) = JsonLogEntry.ValidateJsonLogEntry(messageBody, log);
//Validate Trigram - compare entry from the json log with the trigram azure table storage
(validTrigram, webhookUrl) = Trigram.ValidateTrigram(storageConnectionString, logEntry, trigramTableName, log);
if (validJson)
{
infoMessage = $"Task Run: Json Schema valid: {messageBody}";
log?.LogInformation(infoMessage);
if (validTrigram)
{
//Insert into ElasticSearch Cluster
var elasticLowLevelClient = emp_elastic_operations.ElasticConnect(elasticSearchClusterURI, elasticSearchAPIId, elasticSearchAPIKey, log);
await emp_elastic_operations.ElasticPutAsync(elasticLowLevelClient, logEntry, log);
}
else
{
// If not successful send to the Azure Storage Teams queue
log?.LogInformation($"Task Run: Application Trigram NOT valid: {messageBody}");
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = $"Invalid trigram", LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
}
}
else
{
// If not valid send to the Azure Storage Teams queue
infoMessage = $"Task Run: Json Schema NOT valid: {messageBody}";
log?.LogInformation(infoMessage);
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = errorValidation, LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
}
await Task.Yield();
}
}
catch (Exception ex)
{
if (logEntry != null)
{
// send to the Azure Storage teams queue
log?.LogInformation($"Task Run: exception raised {ex}");
CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log);
var logQueue = new QueueLog()
{
ErrorMessage = $"{ex}", LogEntry = logEntry, WebhookUrl = webhookUrl
};
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log);
// send to the Azure Storage Error queue
log?.LogInformation($"Task Run: exception raised {ex}");
cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, errorQueueName, log);
AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, messageBody, log);
}
}
}
// Once processing of the batch is complete, if any messages in the batch failed processing throw an exception so that there is a record of the failure.
if (exceptions.Count > 1)
{
throw new AggregateException(exceptions);
}
if (exceptions.Count == 1)
{
throw exceptions.Single();
}
}
public void FailJsonWithUndeserializableMessage()
{
Assert.Throws <JsonReaderException>(() => JsonLogEntry.ValidateJsonLogEntry("Undeserializable message", null));
}
public void FailJsonWithMinimalRequirements()
{
var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(NonValidMessage, null);
Assert.False(emp);
}
public void ValidateJsonWithMinimalRequirements()
{
var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(ValidMessage, null);
Assert.True(emp);
}
public void ValidateDeserializeJsonLogEntryWithMinimalRequirementsFailure()
{
Assert.Throws <Newtonsoft.Json.JsonReaderException>(() => JsonLogEntry.DeserializeJsonLogEntry("Non deserialisable message", null));
}
public void ValidateDeserializeJsonLogEntryWithMinimalRequirements()
{
JsonLogEntry emp = JsonLogEntry.DeserializeJsonLogEntry(ValidMessage, null);
Assert.NotNull(emp);
}
/// <inheritdoc /> ValueTask IDataTransferObject.WriteToAsync <TWriter>(TWriter writer, CancellationToken token) => JsonLogEntry.SerializeAsync <T, TWriter>(writer, TypeId, Content, token);
/// <summary>
/// Method: ElasticPut
/// Goal: Sends data to the ElasticSearch Cluster
/// </summary>
/// <param name="lowlevelClient">The ElasticSearch LowLevelClient Object, returned after the connection is made</param>
/// <param name="jsonLogEntry">The emp_json_log_entry object entry with the representation of the log message</param>
/// <param name="log">The ILogger object to log information and errors</param>
/// <returns></returns>
/// <exception cref="UnexpectedElasticsearchClientException">Unknown exceptions, for instance a response from Elasticsearch not properly deserialized. These are sometimes bugs and should be reported.</exception>
/// <exception cref="ElasticsearchClientException">
/// Known exceptions that includes: max retries or timeout reached, bad authentication, etc.
/// Elasticsearch itself returned an error (could not parse the request, bad query, missing field, etc.
///</exception>
public static async Task ElasticPutAsync(ElasticLowLevelClient lowlevelClient, JsonLogEntry jsonLogEntry, ILogger log)
{
try
{
string indexString;
DateTime dt = Convert.ToDateTime(jsonLogEntry.Date);
string strYear = (dt.Year.ToString());
indexString = $"apps_{jsonLogEntry.Trigram.ToLower()}_{strYear}";
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>(indexString, PostData.Serializable(jsonLogEntry));
string indexResponse = asyncIndexResponse.Body;
var success = asyncIndexResponse.Success;
if (success)
{
log?.LogInformation($"ElasticPut: Log entry successfully sent to Elastic Cluster {asyncIndexResponse.Body}");
}
else
{
log?.LogInformation($"ElasticPut: An exception was thrown by the Elastic method {asyncIndexResponse.OriginalException}");
throw new Exception("Could not write in elastic");
}
}
catch (UnexpectedElasticsearchClientException e)
{
log?.LogError($"ElasticPut: UnexpectedElasticsearchClientException : {e.Message}");
throw e;
}
catch (ElasticsearchClientException e)
{
log?.LogError($"ElasticPut: ElasticsearchClientException : {e.Message}");
throw e;
}
}
