private static void SerializeMetricListToWriter(PayloadQueue queue, List <BosunMetric> metrics, DateTime timestamp, ref int metricsCount, ref int bytesWritten) { if (metrics.Count == 0) { return; } var writer = queue.GetWriter(); try { foreach (var m in metrics) { m.SerializeInternal(writer, timestamp); if (queue.IsFull) { break; } } metricsCount += writer.MetricsCount; bytesWritten += writer.TotalBytesWritten; } finally { writer.EndBatch(); } }
public void ConstructionTest() { using (var logger = new RollbarLogger(false)) { PayloadQueue pq = new PayloadQueue(logger); Assert.IsNotNull(pq.Logger); Assert.AreSame(logger, pq.Logger); } }
public WebhookReceiverTests() { _payloadQueue = new PayloadQueue(); _payloadQueue.Clear(); var server = new TestServer(new WebHostBuilder().UseStartup <Startup>()); _client = server.CreateClient(); }
private bool FlushPayload(string path, PayloadQueue queue) { var payload = queue.DequeuePendingPayload(); if (payload == null) { return(false); } var metricsCount = payload.MetricsCount; var bytes = payload.Used; Debug.WriteLine($"BosunReporter: Flushing metrics batch. {metricsCount} metrics. {bytes} bytes."); var info = new AfterPostInfo(); var timer = new StopwatchStruct(); try { timer.Start(); PostToBosun(path, true, sw => sw.Write(payload.Data, 0, payload.Used)); timer.Stop(); queue.ReleasePayload(payload); PostSuccessCount++; TotalMetricsPosted += payload.MetricsCount; return(true); } catch (Exception ex) { timer.Stop(); // posting to Bosun failed, so put the batch back in the queue to try again later Debug.WriteLine("BosunReporter: Posting to the Bosun API failed. Pushing metrics back onto the queue."); PostFailCount++; info.Exception = ex; queue.AddPendingPayload(payload); throw; } finally { // don't use the payload variable in this block - it may have been released back to the pool by now info.Count = metricsCount; info.BytesWritten = bytes; info.MillisecondsDuration = timer.GetElapsedMilliseconds(); LastPostInfo = info; // Use BeginInvoke here to invoke the event listeners asynchronously. // We're inside a lock, so calling the listeners synchronously would put us at risk of a deadlock. AfterPost?.BeginInvoke(info, s_asyncNoopCallback, null); } }
/// <summary> /// Instantiates a new collector (the primary class of BosunReporter). You should typically only instantiate one collector for the lifetime of your /// application. It will manage the serialization of metrics and sending data to Bosun. /// </summary> /// <param name="options"></param> /// <param name="exceptionHandler"></param> public MetricsCollector(BosunOptions options, Action <Exception> exceptionHandler) { if (exceptionHandler == null) { throw new ArgumentNullException(nameof(exceptionHandler)); } ExceptionHandler = exceptionHandler; _localMetricsQueue = new PayloadQueue(QueueType.Local); _externalCounterQueue = new PayloadQueue(QueueType.ExternalCounters); _localMetricsQueue.PayloadDropped += OnPayloadDropped; _externalCounterQueue.PayloadDropped += OnPayloadDropped; // these two setters actually update the queues themselves MaxPayloadSize = options.MaxPayloadSize; MaxPendingPayloads = options.MaxPendingPayloads; MetricsNamePrefix = options.MetricsNamePrefix ?? ""; if (MetricsNamePrefix != "" && !BosunValidation.IsValidMetricName(MetricsNamePrefix)) { throw new Exception("\"" + MetricsNamePrefix + "\" is not a valid metric name prefix."); } GetBosunUrl = options.GetBosunUrl; BosunUrl = GetBosunUrl == null ? options.BosunUrl : GetBosunUrl(); _accessToken = options.AccessToken; _getAccessToken = options.GetAccessToken; ThrowOnPostFail = options.ThrowOnPostFail; ThrowOnQueueFull = options.ThrowOnQueueFull; ReportingInterval = options.ReportingInterval; EnableExternalCounters = options.EnableExternalCounters; PropertyToTagName = options.PropertyToTagName; TagValueConverter = options.TagValueConverter; DefaultTags = ValidateDefaultTags(options.DefaultTags); // start continuous queue-flushing _flushTimer = new Timer(Flush, true, 1000, 1000); // start reporting timer var interval = TimeSpan.FromSeconds(ReportingInterval); _reportingTimer = new Timer(Snapshot, true, interval, interval); // metadata timer - wait 30 seconds to start (so there is some time for metrics to be delcared) if (options.MetaDataReportingInterval > 0) { _metaDataTimer = new Timer(PostMetaData, true, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(options.MetaDataReportingInterval)); } }
public void ConstructionTest() { using (var logger = new RollbarLogger()) { RollbarClient client = new RollbarClient(logger); PayloadQueue pq = new PayloadQueue(logger, client); Assert.IsNotNull(pq.Logger); Assert.AreSame(logger, pq.Logger); Assert.AreSame(client, pq.Client); Assert.AreSame(client.Config, pq.Client.Config); Assert.AreSame(client.Config, logger.Config); } }
/*The PipeMessage method is what will connect to the IoT Edge Hub and will queue the shelf data * for processing. * * It will be registered as an inputMessageHandler with the IoT Edge Hub in the ExecuteAsync method * that follows. */ public async Task <MessageResponse> PipeMessage(Message message, object userContext) { var moduleClient = userContext as ModuleClient; if (moduleClient == null) { throw new InvalidOperationException("UserContext doesn't contain " + "expected values"); } /*This section receives the data from the IoT Edge Hub and converts it to a Shelf object * for processing through signalR to the WebApp module. */ byte[] messageBytes = message.GetBytes(); string messageString = Encoding.UTF8.GetString(messageBytes); if (!string.IsNullOrEmpty(messageString)) { try { _logger.LogInformation($"Receiving Message: {messageString.TrimStart('"').TrimEnd('"').Replace('\\', ' ')}"); Payload productData = JsonConvert.DeserializeObject <Payload> (messageString.TrimStart('"').TrimEnd('"').Replace("\\", String.Empty)); PayloadQueue.QueuePayload(productData); if (PayloadQueue.Count() > 1) { while (PayloadQueue.Count() > 1) { await PayloadQueue.DequeueAsync(new CancellationToken()); //throw away result _logger.LogInformation("Dequeue extra data"); } } // catch and swallow exceptions } catch (AggregateException ex) { _logger.LogError($"Error processing message: {ex.Flatten()}"); } catch (Exception ex) { _logger.LogError($"Error processing message: {ex}"); } } return(MessageResponse.Completed); }
public WebhookReceiverTests() { var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .Build(); var payloadQueueSettings = configuration.GetSection("PayloadQueueSettings").Get <PayloadQueueSettings>(); _payloadQueue = new PayloadQueue(payloadQueueSettings); _payloadQueue.Clear(); var server = new TestServer(new WebHostBuilder().UseConfiguration(configuration).UseStartup <Startup>()); _client = server.CreateClient(); }
/// <summary> /// Instantiates a new collector (the primary class of BosunReporter). You should typically only instantiate one collector for the lifetime of your /// application. It will manage the serialization of metrics and sending data to Bosun. /// </summary> /// <param name="options"></param> public MetricsCollector(BosunOptions options) { ExceptionHandler = options.ExceptionHandler; if (options.SnapshotInterval < TimeSpan.FromSeconds(1)) { throw new InvalidOperationException("options.SnapshotInterval cannot be less than one second"); } _localMetricsQueue = new PayloadQueue(QueueType.Local); _externalCounterQueue = new PayloadQueue(QueueType.ExternalCounters); _localMetricsQueue.PayloadDropped += OnPayloadDropped; _externalCounterQueue.PayloadDropped += OnPayloadDropped; // these two setters actually update the queues themselves MaxPayloadSize = options.MaxPayloadSize; MaxPendingPayloads = options.MaxPendingPayloads; MetricsNamePrefix = options.MetricsNamePrefix ?? ""; if (MetricsNamePrefix != "" && !BosunValidation.IsValidMetricName(MetricsNamePrefix)) { throw new Exception("\"" + MetricsNamePrefix + "\" is not a valid metric name prefix."); } _getUrlDynamic = options.GetBosunUrl; _fixedUrl = options.BosunUrl; _accessToken = options.AccessToken; _getAccessToken = options.GetAccessToken; ThrowOnPostFail = options.ThrowOnPostFail; ThrowOnQueueFull = options.ThrowOnQueueFull; ReportingInterval = options.SnapshotInterval; EnableExternalCounters = options.EnableExternalCounters; PropertyToTagName = options.PropertyToTagName; TagValueConverter = options.TagValueConverter; DefaultTags = ValidateDefaultTags(options.DefaultTags); // start continuous queue-flushing _flushTimer = new Timer(Flush, true, 1000, 1000); // start reporting timer _reportingTimer = new Timer(Snapshot, true, ReportingInterval, ReportingInterval); }
public bool TryFilterNewPayload(T payload) { if (PayloadQueue.Contains(payload)) { return(false); } else if (PayloadQueue.Count < MaxQueueItems) { PayloadQueue.Enqueue(payload); return(true); } else { PayloadQueue.TryDequeue(out _); PayloadQueue.Enqueue(payload); return(true); } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { /*hubContext allows ClientNotifier to act as a part of the signalR hub and send messages. See this * for more information: https://docs.microsoft.com/en-us/aspnet/core/signalr/hubcontext?view=aspnetcore-2.1 */ var notifytask = Task.Run(async() => { while (!stoppingToken.IsCancellationRequested) { if (PayloadQueue.Count() > 0) { await this.HubContext.Clients.All.SendAsync("NewData"); } await Task.Delay(TimeSpan.FromSeconds(1)); } }, stoppingToken); await Task.WhenAll(notifytask); }
void FlushPayloadQueue(PayloadQueue queue, Uri url) { if (queue.PendingPayloadsCount == 0) { return; } if (!ShutdownCalled && queue.SuspendFlushingUntil > DateTime.UtcNow) { return; } while (queue.PendingPayloadsCount > 0) { if (!FlushPayload(url, queue)) { break; } } }
public async Task <bool> TryFilterNewPayload(T payload, Func <T, string> keyExpression) { var payloadKey = $"{typeof(T).Name}:{keyExpression(payload)}"; using (await _payloadLock.WaitAsync(payloadKey)) { if (PayloadQueue.Contains(payload)) { return(false); } else if (PayloadQueue.Count < MaxQueueItems) { PayloadQueue.Enqueue(payload); return(true); } else { PayloadQueue.TryDequeue(out _); PayloadQueue.Enqueue(payload); return(true); } } }
public void AddPayload(int blockNumber, ByteString payloadItemToAdd) { PayloadQueue.Enqueue(new PayloadItem(blockNumber, payloadItemToAdd)); }