private void WriteEventSourceEvent(EventHolder eventHolder) { var eventData = eventHolder.Data as EventSourceEvent; if (eventData?.Entry == null || mLoggingLevel > LoggingLevel.Info) { return; } try { var eventTelemetry = AddTelemetryContext(new EventTelemetry($"EventSource:{eventData.Entry.EntityType}:{eventData.Entry.EventType}"), eventHolder); AddPropertyData(eventTelemetry, nameof(eventData.Entry.EntityType), eventData.Entry.EntityType); AddPropertyData(eventTelemetry, nameof(eventData.Entry.EventType), eventData.Entry.EventType); AddPropertyData(eventTelemetry, nameof(eventData.OriginatorId), eventData.OriginatorId); AddPropertyData(eventTelemetry, nameof(eventData.Entry.CorrelationId), eventData.Entry.CorrelationId); AddPropertyData(eventTelemetry, nameof(eventData.Entry.EntitySource), eventData.Entry.EntitySource); AddPropertyData(eventTelemetry, nameof(eventData.Entry.Key), eventData.Entry.Key); AddPropertyData(eventTelemetry, nameof(eventData.TraceId), eventData.TraceId); mTelemetry?.TrackEvent(eventTelemetry); } catch (Exception ex) { LogTelemetryException(ex); } }
/// <summary> /// This is the extended write method for collectors that require the additional data. /// </summary> /// <param name="holder"></param> public virtual void Write(EventHolder holder) { if (IsSupported(holder.DataType)) { mSupported[holder.DataType](holder); } }
/// <summary> /// Make an Id for a boundary event /// </summary> /// <param name="ev"></param> /// <param name="msId"></param> /// <returns></returns> public static string BoundaryMakeId(EventHolder ev, MicroserviceId msId) { var e = ev.Data as BoundaryEvent; string id = $"{e.Payload?.Message?.ProcessCorrelationKey ?? "NoCorrId"}_{e.Id}"; return($"{string.Join("_", id.Split(Path.GetInvalidFileNameChars()))}.json"); }
private void WriteSecurityEvent(EventHolder eventHolder) { var eventData = eventHolder.Data as SecurityEvent; if (eventData == null) { return; } try { var traceId = new KeyValuePair <string, string>(nameof(eventData.TraceId), eventData.TraceId); if (eventData.Ex != null) { mTelemetry?.TrackException(AddTelemetryContext(new ExceptionTelemetry(eventData.Ex) { Message = $"Security Event - {eventData.Direction}", Properties = { traceId } }, eventHolder)); } else { mTelemetry?.TrackEvent(AddTelemetryContext(new EventTelemetry($"{LoggingLevel.Error}:{eventData.Direction}") { Properties = { traceId } }, eventHolder)); } } catch (Exception ex) { LogTelemetryException(ex); } }
/// <summary> /// This method writes the incoming event data to the data collectors. /// </summary> /// <param name="eventData">The event data.</param> /// <param name="support">The event data type.</param> /// <param name="sync">Specifies whether the data should be written out immediately.</param> /// <param name="claims">The optional claims of the calling party. If not set explicitly, then this /// will be populated from the current thread. If you don't want this then pass an empty claims object.</param> public void Write(EventBase eventData, DataCollectionSupport support, bool sync = false, ClaimsPrincipal claims = null) { if (eventData == null) { throw new ArgumentNullException("eventData", "eventData cannot be null for Write"); } if (!Active) { throw new ServiceNotStartedException(); } //Create the event holder and set the identity based on the claims passed or if null, //picked up from the current thread. var item = new EventHolder(support, claims ?? Thread.CurrentPrincipal as ClaimsPrincipal) { Data = eventData , Sync = sync , Timestamp = StatisticsInternal.ActiveIncrement() }; //Do we have to write this straight away, or can we push it on to an async thread. if (item.Sync) { //Process the item immediately. ProcessItem(item); } else { mQueue.Enqueue(item); //Signal to the logging thread that there is more data waiting. mReset.Set(); } }
/// <summary> /// This method processes the incoming event by looping through the collectors for the particular type. /// </summary> /// <param name="eventData">The event data holder.</param> protected void ProcessItem(EventHolder eventData) { mCollectorSupported[eventData.DataType]? .ForEach((l) => ProcessItem(l, eventData)); //Decrement the active count with the time needed to process. StatisticsInternal.ActiveDecrement(eventData.Timestamp); }
private void WriteBoundaryEvent(EventHolder eventHolder) { var eventData = eventHolder.Data as BoundaryEvent; if (eventData == null || (mLoggingLevel > LoggingLevel.Info && eventData.Ex == null)) { return; } try { EventTelemetry eventTelemetry = null; ExceptionTelemetry exceptionTelemetry = null; ISupportProperties telemetryProperties; // If we have en exception log as such if (eventData.Ex != null) { telemetryProperties = exceptionTelemetry = AddTelemetryContext(new ExceptionTelemetry(eventData.Ex) { Message = eventData.Ex.Message }, eventHolder); } else { telemetryProperties = eventTelemetry = AddTelemetryContext(new EventTelemetry($"Boundary:{eventData.Payload?.Message?.ChannelId}:{eventData.Payload?.Message?.MessageType}:{eventData.Payload?.Message?.ActionType}:{eventData.Direction}"), eventHolder); } AddPropertyData(telemetryProperties, nameof(eventData.Payload.Message.ChannelId), eventData.Payload?.Message?.ChannelId); AddPropertyData(telemetryProperties, nameof(eventData.Payload.Message.MessageType), eventData.Payload?.Message?.MessageType); AddPropertyData(telemetryProperties, nameof(eventData.Payload.Message.ActionType), eventData.Payload?.Message?.ActionType); AddPropertyData(telemetryProperties, nameof(eventData.Payload.Message.CorrelationKey), eventData.Payload?.Message?.CorrelationKey); AddPropertyData(telemetryProperties, nameof(eventData.Payload.Message.FabricDeliveryCount), eventData.Payload?.Message?.FabricDeliveryCount.ToString()); AddPropertyData(telemetryProperties, nameof(BoundaryEventType), eventData.Type.ToString()); // If we have the payload and a correlation key use this as the operation id var telemetry = (ITelemetry)eventTelemetry ?? exceptionTelemetry; if (telemetry != null) { telemetry.Context.Operation.Id = eventData.Payload?.Message?.ProcessCorrelationKey ?? telemetry.Context.Operation.Id; } if (exceptionTelemetry != null) { mTelemetry?.TrackException(exceptionTelemetry); } else if (eventTelemetry != null) { eventTelemetry.Metrics[$"{nameof(BoundaryEvent)}:{nameof(eventData.Requested)}"] = eventData.Requested; eventTelemetry.Metrics[$"{nameof(BoundaryEvent)}:{nameof(eventData.Actual)}"] = eventData.Actual; mTelemetry?.TrackEvent(eventTelemetry); } } catch (Exception ex) { LogTelemetryException(ex); } }
/// <summary> /// This method serializes incoming objects in to standard JSON format encoded as UTF8. /// </summary> /// <param name="e">The incoming EventBase.</param> /// <param name="id">The Microservice Id</param> /// <returns>Returns the byte array.</returns> public static AzureStorageBinary DefaultJsonBinarySerializer(EventHolder e, MicroserviceId id) { var jObj = JObject.FromObject(e.Data); var body = jObj.ToString(); return(new AzureStorageBinary { Blob = Encoding.UTF8.GetBytes(body) }); }
/// <summary> /// This method is called after an event has been processed. /// It will be passed to any event listeners for additional processing. /// </summary> /// <param name="eventData">The event data.</param> protected virtual void RaiseEvent(EventHolder eventData) { try { OnEvent?.Invoke(this, eventData); } catch (Exception) { //No exceptions here } }
/// <summary> /// This method writes the event holder to table storage. /// </summary> /// <param name="e">The event holder to write to table storage.</param> /// <param name="id">The service id.</param> /// <returns>This is an async process.</returns> public override async Task Write(EventHolder e, MicroserviceId id) { //Create the output. var output = Serializer(e, id); // Create the TableOperation object that inserts the customer entity. TableOperation insert = TableOperation.Insert(output); // Execute the insert operation. await Table.ExecuteAsync(insert); }
private void Action(EventHolder holder) { try { mAction(Context, holder); } catch (Exception ex) { //We're not interested in catching exceptions here. This is for the pipeline code to sort out. } }
public static ITableEntity ToTableTelemetryEvent(EventHolder e, MicroserviceId msId) { var ev = e.Data as TelemetryEvent; var dict = new Dictionary <string, EntityProperty>(); dict.Add("Metric", new EntityProperty(ev.MetricName)); dict.Add("Value", new EntityProperty(ev.Value)); return(new DynamicTableEntity("Telemetry" + DatePartition(), ev.TraceId, "*", dict)); }
/// <summary> /// This method wraps the individual request in a safe wrapper. /// </summary> /// <param name="dataCollector">The data collector component.</param> /// <param name="eventData">The event data holder.</param> protected virtual void ProcessItem(IDataCollectorComponent dataCollector, EventHolder eventData) { try { dataCollector.Write(eventData); } catch (Exception ex) { //We don't want unexpected exceptions here and to stop the other loggers working. StatisticsInternal.ErrorIncrement(); StatisticsInternal.Ex = ex; } }
public static string LoggerMakeFolder(EventHolder ev, MicroserviceId msId) { var e = ev.Data as LogEvent; string level = Enum.GetName(typeof(LoggingLevel), e.Level); return(string.Format("{0}/{1}/{2:yyyy-MM-dd}/{2:HH}", msId.Name, level, DateTime.UtcNow)); //if (e is ILogStoreName) // return ((ILogStoreName)logEvent).StorageId; //// If there is a category specified and it contains valid digits or characters then make it part of the log name to make it easier to filter log events //if (!string.IsNullOrEmpty(logEvent.Category) && logEvent.Category.Any(char.IsLetterOrDigit)) // return string.Format("{0}_{1}_{2}", logEvent.GetType().Name, new string(logEvent.Category.Where(char.IsLetterOrDigit).ToArray()), Guid.NewGuid().ToString("N")); //return string.Format("{0}_{1}", logEvent.GetType().Name, Guid.NewGuid().ToString("N")); }
public static ITableEntity ToTableDispatcherEvent(EventHolder e, MicroserviceId msId) { var ev = e.Data as DispatcherEvent; var dict = new Dictionary <string, EntityProperty>(); dict.Add("IsSuccess", new EntityProperty(ev.IsSuccess)); dict.Add("Type", GetEnum <PayloadEventType>(ev.Type)); dict.Add("Reason", GetEnum <DispatcherRequestUnresolvedReason>(ev.Reason)); dict.Add("Delta", new EntityProperty(ev.Delta)); dict.Add("Ex", new EntityProperty(ev.Ex?.Message)); dict.AddPayloadCommon(ev.Payload); //ETag: Set this value to '*' to blindly overwrite an entity as part of an update operation. return(new DynamicTableEntity("Dispatcher" + DatePartition(), ev.TraceId, "*", dict)); }
private void WriteLogEvent(EventHolder eventHolder) { var eventData = eventHolder.Data as LogEvent; if (eventData == null || eventData.Level < mLoggingLevel) { return; } try { EventTelemetry eventTelemetry = null; ExceptionTelemetry exceptionTelemetry = null; ISupportProperties telemetryProperties; // Don't log non errors that have exceptions as exceptions i.e. warnings / info if (eventData.Ex != null && eventData.Level >= LoggingLevel.Error) { telemetryProperties = exceptionTelemetry = AddTelemetryContext(new ExceptionTelemetry(eventData.Ex), eventHolder); } else { telemetryProperties = eventTelemetry = AddTelemetryContext(new EventTelemetry(eventData.Level + (!string.IsNullOrEmpty(eventData.Category) ? $":{eventData.Category}" : string.Empty)), eventHolder); } AddPropertyData(telemetryProperties, nameof(LoggingLevel), eventData.Level.ToString()); AddPropertyData(telemetryProperties, nameof(eventData.TraceId), eventData.TraceId); AddPropertyData(telemetryProperties, nameof(eventData.Message), eventData.Message); AddPropertyData(telemetryProperties, nameof(eventData.Category), eventData.Category); AddPropertyData(telemetryProperties, "Exception.Message", eventData.Ex?.Message); eventData.AdditionalData?.ForEach(kvp => AddPropertyData(telemetryProperties, kvp.Key, kvp.Value)); if (exceptionTelemetry != null) { mTelemetry?.TrackException(exceptionTelemetry); } else { AddPropertyData(telemetryProperties, nameof(Exception), eventData.Ex?.ToString()); mTelemetry?.TrackEvent(eventTelemetry); } } catch (Exception ex) { LogTelemetryException(ex); } }
/// <summary> /// Write event source /// </summary> /// <param name="support"></param> /// <param name="eventHolder"></param> protected virtual void WriteEvent(DataCollectionSupport support, EventHolder eventHolder) { var options = mPolicy.Options.FirstOrDefault(o => o.Support == support); if (options == null || !options.IsSupported(eventHolder)) { return; } int start = StatisticsInternal.ActiveIncrement(options.Support); Guid?traceId = options.ShouldProfile ? (ProfileStart($"AzureEventHub{options.Support}_{eventHolder.Data.TraceId}")) : default(Guid?); // Check we have en event hub client mEventHubClient = mEventHubClient ?? CreateEventHubClient(); var result = ResourceRequestResult.Unknown; try { //Serialize the blob. var blob = options.SerializerBinary(eventHolder, OriginatorId).Blob; //Encrypt the blob is that is the policy. if (options.EncryptionPolicy != AzureStorageEncryption.None && mEncryption != null) { blob = ServiceHandlers.Encryption[mEncryption].Encrypt(blob); } mEventHubClient.SendAsync(new EventData(blob)).Wait(); result = ResourceRequestResult.Success; } catch (Exception) { result = ResourceRequestResult.Exception; StatisticsInternal.ErrorIncrement(options.Support); throw; } finally { StatisticsInternal.ActiveDecrement(options.Support, start); if (traceId.HasValue) { ProfileEnd(traceId.Value, start, result); } } }
/// <summary> /// This method writes to the incoming event to the underlying storage technology. /// </summary> /// <param name="e">The event.</param> /// <param name="id">The microservice metadata.</param> /// <returns> /// This is an async task. /// </returns> public override async Task Write(EventHolder e, MicroserviceId id) { var output = Serializer(e, id); //Encrypt the payload when required. if (EncryptionPolicy != AzureStorageEncryption.None && Encryptor != null) { //The checks for always encrypt are done externally. output.Blob = Encryptor(output.Blob); } // Create a message and add it to the queue. CloudQueueMessage message = new CloudQueueMessage(Convert.ToBase64String(output.Blob)); // Async enqueue the message await Queue.AddMessageAsync(message); }
private void WriteTelemetryEvent(EventHolder eventHolder) { var eventData = eventHolder.Data as TelemetryEvent; if (string.IsNullOrEmpty(eventData?.MetricName)) { return; } try { mTelemetry?.TrackMetric(eventData.MetricName, eventData.Value, eventData.AdditionalData); } catch (Exception ex) { LogTelemetryException(ex); } }
public static ITableEntity ToTableBoundaryEvent(EventHolder e, MicroserviceId msId) { var ev = e.Data as BoundaryEvent; var dict = new Dictionary <string, EntityProperty>(); dict.Add("Type", GetEnum <BoundaryEventType>(ev.Type)); dict.Add("Direction", GetEnum <ChannelDirection>(ev.Direction)); dict.Add("ChannelId", new EntityProperty(ev.ChannelId)); dict.Add("ChannelPriority", new EntityProperty(ev.ChannelPriority)); dict.Add("Id", new EntityProperty(ev.Id)); dict.Add("BatchId", new EntityProperty(ev.BatchId)); dict.Add("Requested", new EntityProperty(ev.Requested)); dict.Add("Actual", new EntityProperty(ev.Actual)); dict.Add("Ex", new EntityProperty(ev.Ex?.Message)); dict.AddPayloadCommon(ev.Payload); return(new DynamicTableEntity("Boundary" + DatePartition(), ev.TraceId, "*", dict)); }
public static ITableEntity ToTableLogEvent(EventHolder e, MicroserviceId id) { var ev = e.Data as LogEvent; var dict = new Dictionary <string, EntityProperty>(); dict.Add("Machine", new EntityProperty(id.MachineName)); dict.Add("Name", new EntityProperty(id.Name)); dict.Add("ServiceId", new EntityProperty(id.ServiceId)); dict.Add("Level", GetEnum <LoggingLevel>(ev.Level)); dict.Add("Category", new EntityProperty(ev.Category)); dict.Add("Message", new EntityProperty(ev.Message)); dict.Add("Ex", new EntityProperty(ev.Ex?.Message)); //ETag: Set this value to '*' to blindly overwrite an entity as part of an update operation. return(new DynamicTableEntity("Logger" + DatePartition(), ev.TraceId, "*", dict)); }
/// <summary> /// Update telemetry context based on meta data held in the event holder /// </summary> /// <param name="telemetry"></param> /// <param name="eventHolder"></param> private T AddTelemetryContext <T>(T telemetry, EventHolder eventHolder) where T : ITelemetry { if (telemetry == null || eventHolder?.Claims == null) { return(telemetry); } var correlationClaim = eventHolder.Claims.FindFirst(JwtTokenAuthenticationHandler.ClaimProcessCorrelationKey); if (!string.IsNullOrEmpty(correlationClaim?.Value)) { telemetry.Context.Operation.Id = correlationClaim.Value; } if (!string.IsNullOrEmpty(eventHolder.Claims?.Identity?.Name)) { telemetry.Context.User.AuthenticatedUserId = eventHolder.Claims?.Identity?.Name; } return(telemetry); }
/// <summary> /// This method writes the event holder to table storage. /// </summary> /// <param name="e">The event holder.</param> /// <param name="id">The service identifier class.</param> /// <returns>The is an async task.</returns> public override async Task Write(EventHolder e, MicroserviceId id) { string storageId = MakeId(e, id); string storageFolder = MakeFolder(e, id); var output = Serializer(e, id); //Encrypt the payload when required. if (EncryptionPolicy != AzureStorageEncryption.None && Encryptor != null) { //The checks for always encrypt are done externally. output.Blob = Encryptor(output.Blob); } var refEntityDirectory = Container.GetDirectoryReference(storageFolder); var Blob = refEntityDirectory.GetBlockBlobReference(storageId); Blob.Properties.ContentType = output.ContentType; Blob.Properties.ContentEncoding = output.ContentEncoding; await Blob.UploadFromByteArrayAsync(output.Blob, 0, output.Blob.Length); }
/// <summary> /// This method provides the default logging level support for the types of logs. /// </summary> /// <param name="behaviour">The storage type.</param> /// <param name="e">The log event.</param> /// <returns>Returns true if the message should be logged.</returns> public static bool DefaultLogLevelSupport(AzureStorageBehaviour behaviour, EventHolder ev) { var e = ev.Data as LogEvent; if (e == null) { return(false); } switch (behaviour) { case AzureStorageBehaviour.Table: return(true); case AzureStorageBehaviour.Blob: return(e.Level == LoggingLevel.Fatal || e.Level == LoggingLevel.Error || e.Level == LoggingLevel.Warning); default: return(false); } }
/// <summary> /// Output the data for the three option types. /// </summary> /// <param name="support">The storage options</param> /// <param name="e">The event object.</param> protected void WriteConnectors(DataCollectionSupport support, EventHolder e) { //Blob if (mHoldersBlob.ContainsKey(support) && mHoldersBlob[support].ShouldWrite(e)) { WriteConnector(mHoldersBlob[support], e).Wait(); } //Table if (mHoldersTable.ContainsKey(support) && mHoldersTable[support].ShouldWrite(e)) { WriteConnector(mHoldersTable[support], e).Wait(); } //Queue if (mHoldersQueue.ContainsKey(support) && mHoldersQueue[support].ShouldWrite(e)) { WriteConnector(mHoldersQueue[support], e).Wait(); } //File if (mHoldersFile.ContainsKey(support) && mHoldersFile[support].ShouldWrite(e)) { WriteConnector(mHoldersFile[support], e).Wait(); } }
private void WriteMicroserviceStatistics(EventHolder eventHolder) { var eventData = eventHolder.Data as MicroserviceStatistics; if (string.IsNullOrEmpty(eventData?.Name)) { return; } try { mTelemetry?.TrackMetric($"{eventData.Name}.Tasks.Active", eventData.Tasks?.Availability?.Active ?? 0); mTelemetry?.TrackMetric($"{eventData.Name}.Tasks.SlotsAvailable", eventData.Tasks?.Availability?.SlotsAvailable ?? 0); mTelemetry?.TrackMetric($"{eventData.Name}.Tasks.Killed", eventData.Tasks?.Availability?.Killed ?? 0); mTelemetry?.TrackMetric($"{eventData.Name}.Tasks.KilledDidReturn", eventData.Tasks?.Availability?.KilledDidReturn ?? 0); mTelemetry?.TrackMetric($"{eventData.Name}.Tasks.KilledTotal", (eventData.Tasks?.Availability?.Killed ?? 0) + (eventData.Tasks?.Availability?.KilledDidReturn ?? 0)); mTelemetry?.TrackMetric($"{eventData.Name}.Cpu.ServicePercentage", eventData.Tasks?.Cpu?.ServicePercentage ?? 0); } catch (Exception ex) { LogTelemetryException(ex); } }
/// <summary> /// Identifies whether the event should be output. /// </summary> /// <param name="e">The event holder to write.</param> /// <returns>Returns true if this should be written.</returns> public override bool ShouldWrite(EventHolder e) { return(Options.IsSupported?.Invoke(AzureStorageBehaviour.Blob, e) ?? false); }
/// <summary> /// Output the data for the three option types. /// </summary> /// <param name="support">The storage options</param> /// <param name="e">The event object.</param> protected void WriteConnectors(DataCollectionSupport support, EventHolder e) { List <Exception> exs = null; Action <Action> wrapper = (a) => { try { a(); } catch (Exception ex) { if (exs == null) { exs = new List <Exception>(); } exs.Add(ex); } }; //Blob wrapper(() => { if (mHoldersBlob.ContainsKey(support) && mHoldersBlob[support].ShouldWrite(e)) { WriteConnector(mHoldersBlob[support], e).Wait(); } }); //Table wrapper(() => { if (mHoldersTable.ContainsKey(support) && mHoldersTable[support].ShouldWrite(e)) { WriteConnector(mHoldersTable[support], e).Wait(); } }); //Queue wrapper(() => { if (mHoldersQueue.ContainsKey(support) && mHoldersQueue[support].ShouldWrite(e)) { WriteConnector(mHoldersQueue[support], e).Wait(); } }); //File wrapper(() => { if (mHoldersFile.ContainsKey(support) && mHoldersFile[support].ShouldWrite(e)) { WriteConnector(mHoldersFile[support], e).Wait(); } }); //If there were errors during execution, then throw an aggregrate exception. if (exs != null && exs.Count > 0) { throw new AzureDataCollectionAggregrateException("WriteConnectors failure.", exs); } }
/// <summary> /// This method writes the event data to the underlying storage. /// </summary> /// <param name="connector">The generic connector.</param> /// <param name="e">The event to write.</param> /// <returns>This is an async process.</returns> protected virtual async Task WriteConnector(IAzureStorageConnectorBase connector, EventHolder e) { int start = StatisticsInternal.ActiveIncrement(connector.Support); Guid?traceId = connector.Options.ShouldProfile ? (ProfileStart($"Azure{connector.Support}_{e.Data.TraceId}")) : default(Guid?); var result = ResourceRequestResult.Unknown; try { await connector.Write(e, OriginatorId); result = ResourceRequestResult.Success; } catch (StorageThrottlingException) { result = ResourceRequestResult.Exception; throw; } catch (Exception ex) { result = ResourceRequestResult.Exception; //Collector?.LogException(string.Format("Unable to output {0} to {1} for {2}", id, directory, typeof(E).Name), ex); StatisticsInternal.ErrorIncrement(connector.Support); throw; } finally { StatisticsInternal.ActiveDecrement(connector.Support, start); if (traceId.HasValue) { ProfileEnd(traceId.Value, start, result); } } }
/// <summary> /// This method is used to check that the specific event should be written to the underlying storage. /// </summary> /// <param name="e">The event.</param> /// <returns> /// Returns true if the event should be written. /// </returns> public override bool ShouldWrite(EventHolder e) { return(Options.IsSupported(AzureStorageBehaviour.Queue, e)); }