/// <summary> /// Function callback to post a message in Teams. /// TODO: work on the "message" parameter, so far just a simple text mokup /// </summary> /// <remarks> /// The object posted should be like this: /// { /// "message":"error test with objects", /// "emp_json_log_entry": { /// "trigram": "abc", /// "application": "Axaim.Membership.Admin", /// "layer": "lifetest", /// "level": "DEBUG", /// "date": "Z2020etcetc", /// "message": "the original message", /// "WebhookURL": "" /// } /// } /// TODO: adjust this based on the real emp_json_log_entry class /// </remarks> /// <param name="req">The http request</param> /// <param name="log">Logger to log</param> /// <returns>a result action wit OK if all goes right or BadRequestObjectResult in case of issue</returns> public static async Task <IActionResult> RunFromDataString(string requestBody, ILogger log) { // Get the message from the body string message; dynamic data; try { data = JsonConvert.DeserializeObject(requestBody); message = data?.message; } catch (JsonReaderException ex) { log?.LogError(ex, "Error deserializing the message"); return((ActionResult) new BadRequestObjectResult($"Error: {ex}")); } log?.LogInformation("Processing a request to post an error on a Teams channel"); if (message == null) { log?.LogError("No error message provided"); return(new BadRequestObjectResult("Please pass a name on the query string or in the request body")); } // Get the emp_json_log_entry from the body JsonLogEntry logEntry = new JsonLogEntry(); logEntry.Trigram = data?.emp_log_entry?.trigram; logEntry.Level = data?.emp_log_entry?.level; logEntry.Date = data?.emp_log_entry?.date; logEntry.Message = data?.emp_log_entry?.message; string webhookURL = data?.WebhookURL; return(await PostOnTeams(message, webhookURL, logEntry, log)); }
public void ValidVariousDates() { Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9218638Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.92Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.921Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.9218Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.92186Z")); Assert.True(JsonLogEntry.IsValidIso8601Date("2020-02-11T17:38:36.921863Z")); Assert.False(JsonLogEntry.IsValidIso8601Date(null)); Assert.False(JsonLogEntry.IsValidIso8601Date(string.Empty)); Assert.False(JsonLogEntry.IsValidIso8601Date("Non date format")); Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11")); Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11T11:15")); Assert.False(JsonLogEntry.IsValidIso8601Date("2020-02-11T11:15Z")); }
protected System.Collections.Generic.IEnumerable <JsonLogEntry> GetEntries() { Newtonsoft.Json.JsonSerializer oJsonSerializer = new Newtonsoft.Json.JsonSerializer(); foreach (System.Tuple <string, System.IO.Stream> st in StreamFactory.GetStreams(From, this.Recurse)) { this.StreamLineNumber = 0; this.CurrentStreamName = st.Item1; this.RecordIndex = 0; using (AdvancedTextReader srdr = new AdvancedTextReader(st.Item2, this.TextReaderOptions)) { using (var m_JsonReader = new Newtonsoft.Json.JsonTextReader(srdr) { SupportMultipleContent = true }) { bool bStartRead = string.IsNullOrEmpty(JsonPropertyToRead); while (m_JsonReader.Read()) { if (!bStartRead && !(m_JsonReader.TokenType == JsonToken.PropertyName && m_JsonReader.Value == JsonPropertyToRead)) { continue; } else { bStartRead = true; } if (m_JsonReader.TokenType == JsonToken.StartObject) { // Additional fields this.StreamLineNumber = m_JsonReader.LineNumber; this.RecordIndex += 1; // Deserialize entries and yield JsonLogEntry le = oJsonSerializer.Deserialize <JsonLogEntry>(m_JsonReader); yield return(le); } } } } } }
protected override void InitValueExtractor() { // Init value extractor from first record ValueExtractor = new DictionaryExtractor(new string[] { }); // init empty first if can't init later Newtonsoft.Json.JsonSerializer oJsonSerializer = new Newtonsoft.Json.JsonSerializer(); foreach (var st in StreamFactory.GetStreams(this.From, this.Recurse)) // returns touplse (stream name as string, Stream object implementaion) { using (AdvancedTextReader srdr = new AdvancedTextReader(st.Item2, this.TextReaderOptions)) { using (var m_JsonReader = new Newtonsoft.Json.JsonTextReader(srdr) { SupportMultipleContent = true }) { bool bStartRead = string.IsNullOrEmpty(JsonPropertyToRead); while (m_JsonReader.Read()) { if (!bStartRead && !(m_JsonReader.TokenType == JsonToken.PropertyName && m_JsonReader.Value == JsonPropertyToRead)) { continue; } else { bStartRead = true; } if (m_JsonReader.TokenType == JsonToken.StartObject) { // Deserialize JsonLogEntry logentry = oJsonSerializer.Deserialize <JsonLogEntry>(m_JsonReader); // Set the value extractor from the first JSON record and exit ValueExtractor = new DictionaryExtractor(logentry); break; } } } } } }
/// <summary> /// Create a message card ready to publish in Teams /// TODO: rather than string, need to create a class or have multiple parameters to the message to post /// </summary> /// <param name="message">The message to post see TODO</param> /// <returns>A message card ready to post</returns> public static MessageCard CreateMessageCard(string message, JsonLogEntry logEntry) { MessageCard card = new MessageCard(); card.Title = "Log entry error"; card.Text = $"Error: {message}"; card.Sections = new List <Section> { new Section { ActivityTitle = logEntry?.Trigram, ActivitySubtitle = DateTime.Now.ToString(), Facts = new List <Fact> { new Fact { Name = $"{nameof(logEntry.Date)}", Value = logEntry?.Date }, new Fact { Name = $"{nameof(logEntry.Level)}", Value = logEntry?.Level } }, Text = $"Original message: {logEntry?.Message}" } }; card.Actions = new List <IAction> { new OpenUriAction { Type = ActionType.OpenUri, Name = "View on site", Targets = new List <Target> { new Target { OS = TargetOs.Default, Uri = string.Format(Environment.GetEnvironmentVariable(DefaultKibanaUrl), logEntry?.Trigram) } } } }; return(card); }
public void FailJsonWithEmptyMessageParts() { string message = ValidMessage; // testing date message = message.Replace(@"""date"": ""2020-02-11T17:38:36.9218638Z""", @"""date"": """""); var(emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing trigram message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing trigram message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : """""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing message message = message.Replace(@"""message"" : ""Life Test Requested""", @"""message"" : """""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing level message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEBUG"""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing level must be uppercase message = message.Replace(@"""level"": ""DEBUG""", @"""level"": ""DEbUG"""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); // testing length (must be 3) message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""ABCD"""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); message = message.Replace(@"""trigram"" : ""arc""", @"""trigram"" : ""AD"""); (emp, mess) = JsonLogEntry.ValidateJsonLogEntry(message, null); Assert.False(emp); }
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter) { if (formatter is null) { throw new ArgumentNullException(nameof(formatter)); } var message = new JsonLogEntry { Timestamp = DateTimeOffset.UtcNow, LogLevel = logLevel.ToString(), Category = _categoryName, Exception = exception?.ToString(), Message = formatter(state, exception), }; // Append the data of all BeginScope and LogXXX parameters to the message dictionary // REMOVED: AppendScope(message.Scope, state); - This line in original JsonLogger from Microsoft it's adding duplicated message details AppendScope(message.Scope); _writer.WriteLine(JsonConvert.SerializeObject(message, Formatting.None)); }
/// <summary> /// Post a message in Teams. /// </summary> /// <param name="message">A message</param> /// <param name="webhookURL">The webhook URL</param> /// <param name="logEntry">A log entry</param> /// <param name="log">The logger</param> /// <returns>>a result action wit OK if all goes right or BadRequestObjectResult in case of issue</returns> public static async Task <IActionResult> PostOnTeams(string message, string webhookURL, JsonLogEntry logEntry, ILogger log) { string url = string.IsNullOrEmpty(webhookURL) ? Environment.GetEnvironmentVariable(DefaultWebhookUrlEnvironmenent) : webhookURL; if (string.IsNullOrEmpty(url)) { log?.LogError($"No default {DefaultWebhookUrlEnvironmenent} environment variable provided"); return(new BadRequestObjectResult($"A default WebhookURL needs to be provided")); } MessageCard card; // Prepare the message if (logEntry != null) { card = CreateMessageCard(message, logEntry); } else { card = CreateBasicMessage(message); } var json = JsonConvert.SerializeObject(card); return(await PostOnTeamsMessage(json, url, log)); }
public static async Task Run([EventHubTrigger("%EMP_EVENT_HUB_NAME%", Connection = "EMP_EVENTHUB_CONNECTION_STRING")] EventData[] events, ILogger log, ExecutionContext executionContext) { var exceptions = new List <Exception>(); string infoMessage; string errorMessage; string jsonSettingsPath = executionContext.FunctionAppDirectory; //App Settings from local.settings.json string storageConnectionString = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_CONNECTION_STRING; string teamsQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_TEAMS_QUEUE_NAME; string errorQueueName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_STORAGE_ACCOUNT_ERROR_QUEUE_NAME; string trigramTableName = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_TRIGRAM_TABLE_NAME; string elasticSearchClusterURI = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_CLUSTER_URI; string elasticSearchAPIId = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_ID; string elasticSearchAPIKey = ApSettings.LoadAppSettings(jsonSettingsPath, log).EMP_ELASTIC_SEARCH_API_KEY; foreach (EventData eventData in events) { JsonLogEntry logEntry = null; string webhookUrl = null; bool validTrigram = false; string messageBody = null; try { messageBody = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count); log?.LogInformation($"FROM EVENTHUB {messageBody}"); if (string.IsNullOrEmpty(messageBody)) { infoMessage = $"Task Run: Event is null"; log?.LogInformation(infoMessage); } else { // Validate semantically the json schema logEntry = JsonLogEntry.DeserializeJsonLogEntry(messageBody, log); var(validJson, errorValidation) = JsonLogEntry.ValidateJsonLogEntry(messageBody, log); //Validate Trigram - compare entry from the json log with the trigram azure table storage (validTrigram, webhookUrl) = Trigram.ValidateTrigram(storageConnectionString, logEntry, trigramTableName, log); if (validJson) { infoMessage = $"Task Run: Json Schema valid: {messageBody}"; log?.LogInformation(infoMessage); if (validTrigram) { //Insert into ElasticSearch Cluster var elasticLowLevelClient = emp_elastic_operations.ElasticConnect(elasticSearchClusterURI, elasticSearchAPIId, elasticSearchAPIKey, log); await emp_elastic_operations.ElasticPutAsync(elasticLowLevelClient, logEntry, log); } else { // If not successful send to the Azure Storage Teams queue log?.LogInformation($"Task Run: Application Trigram NOT valid: {messageBody}"); CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log); var logQueue = new QueueLog() { ErrorMessage = $"Invalid trigram", LogEntry = logEntry, WebhookUrl = webhookUrl }; AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log); } } else { // If not valid send to the Azure Storage Teams queue infoMessage = $"Task Run: Json Schema NOT valid: {messageBody}"; log?.LogInformation(infoMessage); CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log); var logQueue = new QueueLog() { ErrorMessage = errorValidation, LogEntry = logEntry, WebhookUrl = webhookUrl }; AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log); } await Task.Yield(); } } catch (Exception ex) { if (logEntry != null) { // send to the Azure Storage teams queue log?.LogInformation($"Task Run: exception raised {ex}"); CloudQueue cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, teamsQueueName, log); var logQueue = new QueueLog() { ErrorMessage = $"{ex}", LogEntry = logEntry, WebhookUrl = webhookUrl }; AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, JsonConvert.SerializeObject(logQueue), log); // send to the Azure Storage Error queue log?.LogInformation($"Task Run: exception raised {ex}"); cloudQueue = AzureStorageQueueOperations.CreateAzureQueue(storageConnectionString, errorQueueName, log); AzureStorageQueueOperations.InsertMessageQueue(cloudQueue, messageBody, log); } } } // Once processing of the batch is complete, if any messages in the batch failed processing throw an exception so that there is a record of the failure. if (exceptions.Count > 1) { throw new AggregateException(exceptions); } if (exceptions.Count == 1) { throw exceptions.Single(); } }
public void FailJsonWithUndeserializableMessage() { Assert.Throws <JsonReaderException>(() => JsonLogEntry.ValidateJsonLogEntry("Undeserializable message", null)); }
public void FailJsonWithMinimalRequirements() { var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(NonValidMessage, null); Assert.False(emp); }
public void ValidateJsonWithMinimalRequirements() { var(emp, message) = JsonLogEntry.ValidateJsonLogEntry(ValidMessage, null); Assert.True(emp); }
public void ValidateDeserializeJsonLogEntryWithMinimalRequirementsFailure() { Assert.Throws <Newtonsoft.Json.JsonReaderException>(() => JsonLogEntry.DeserializeJsonLogEntry("Non deserialisable message", null)); }
public void ValidateDeserializeJsonLogEntryWithMinimalRequirements() { JsonLogEntry emp = JsonLogEntry.DeserializeJsonLogEntry(ValidMessage, null); Assert.NotNull(emp); }
/// <inheritdoc /> ValueTask IDataTransferObject.WriteToAsync <TWriter>(TWriter writer, CancellationToken token) => JsonLogEntry.SerializeAsync <T, TWriter>(writer, TypeId, Content, token);
/// <summary> /// Method: ElasticPut /// Goal: Sends data to the ElasticSearch Cluster /// </summary> /// <param name="lowlevelClient">The ElasticSearch LowLevelClient Object, returned after the connection is made</param> /// <param name="jsonLogEntry">The emp_json_log_entry object entry with the representation of the log message</param> /// <param name="log">The ILogger object to log information and errors</param> /// <returns></returns> /// <exception cref="UnexpectedElasticsearchClientException">Unknown exceptions, for instance a response from Elasticsearch not properly deserialized. These are sometimes bugs and should be reported.</exception> /// <exception cref="ElasticsearchClientException"> /// Known exceptions that includes: max retries or timeout reached, bad authentication, etc. /// Elasticsearch itself returned an error (could not parse the request, bad query, missing field, etc. ///</exception> public static async Task ElasticPutAsync(ElasticLowLevelClient lowlevelClient, JsonLogEntry jsonLogEntry, ILogger log) { try { string indexString; DateTime dt = Convert.ToDateTime(jsonLogEntry.Date); string strYear = (dt.Year.ToString()); indexString = $"apps_{jsonLogEntry.Trigram.ToLower()}_{strYear}"; var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>(indexString, PostData.Serializable(jsonLogEntry)); string indexResponse = asyncIndexResponse.Body; var success = asyncIndexResponse.Success; if (success) { log?.LogInformation($"ElasticPut: Log entry successfully sent to Elastic Cluster {asyncIndexResponse.Body}"); } else { log?.LogInformation($"ElasticPut: An exception was thrown by the Elastic method {asyncIndexResponse.OriginalException}"); throw new Exception("Could not write in elastic"); } } catch (UnexpectedElasticsearchClientException e) { log?.LogError($"ElasticPut: UnexpectedElasticsearchClientException : {e.Message}"); throw e; } catch (ElasticsearchClientException e) { log?.LogError($"ElasticPut: ElasticsearchClientException : {e.Message}"); throw e; } }