private static void SerializeMetricListToWriter(PayloadQueue queue, List <BosunMetric> metrics, DateTime timestamp, ref int metricsCount, ref int bytesWritten)
{
if (metrics.Count == 0)
{
return;
}
var writer = queue.GetWriter();
try
{
foreach (var m in metrics)
{
m.SerializeInternal(writer, timestamp);
if (queue.IsFull)
{
break;
}
}
metricsCount += writer.MetricsCount;
bytesWritten += writer.TotalBytesWritten;
}
finally
{
writer.EndBatch();
}
}
public void ConstructionTest()
{
using (var logger = new RollbarLogger(false))
{
PayloadQueue pq = new PayloadQueue(logger);
Assert.IsNotNull(pq.Logger);
Assert.AreSame(logger, pq.Logger);
}
}
public WebhookReceiverTests()
{
_payloadQueue = new PayloadQueue();
_payloadQueue.Clear();
var server = new TestServer(new WebHostBuilder().UseStartup <Startup>());
_client = server.CreateClient();
}
private bool FlushPayload(string path, PayloadQueue queue)
{
var payload = queue.DequeuePendingPayload();
if (payload == null)
{
return(false);
}
var metricsCount = payload.MetricsCount;
var bytes = payload.Used;
Debug.WriteLine($"BosunReporter: Flushing metrics batch. {metricsCount} metrics. {bytes} bytes.");
var info = new AfterPostInfo();
var timer = new StopwatchStruct();
try
{
timer.Start();
PostToBosun(path, true, sw => sw.Write(payload.Data, 0, payload.Used));
timer.Stop();
queue.ReleasePayload(payload);
PostSuccessCount++;
TotalMetricsPosted += payload.MetricsCount;
return(true);
}
catch (Exception ex)
{
timer.Stop();
// posting to Bosun failed, so put the batch back in the queue to try again later
Debug.WriteLine("BosunReporter: Posting to the Bosun API failed. Pushing metrics back onto the queue.");
PostFailCount++;
info.Exception = ex;
queue.AddPendingPayload(payload);
throw;
}
finally
{
// don't use the payload variable in this block - it may have been released back to the pool by now
info.Count = metricsCount;
info.BytesWritten = bytes;
info.MillisecondsDuration = timer.GetElapsedMilliseconds();
LastPostInfo = info;
// Use BeginInvoke here to invoke the event listeners asynchronously.
// We're inside a lock, so calling the listeners synchronously would put us at risk of a deadlock.
AfterPost?.BeginInvoke(info, s_asyncNoopCallback, null);
}
}
/// <summary>
/// Instantiates a new collector (the primary class of BosunReporter). You should typically only instantiate one collector for the lifetime of your
/// application. It will manage the serialization of metrics and sending data to Bosun.
/// </summary>
/// <param name="options"></param>
/// <param name="exceptionHandler"></param>
public MetricsCollector(BosunOptions options, Action <Exception> exceptionHandler)
{
if (exceptionHandler == null)
{
throw new ArgumentNullException(nameof(exceptionHandler));
}
ExceptionHandler = exceptionHandler;
_localMetricsQueue = new PayloadQueue(QueueType.Local);
_externalCounterQueue = new PayloadQueue(QueueType.ExternalCounters);
_localMetricsQueue.PayloadDropped += OnPayloadDropped;
_externalCounterQueue.PayloadDropped += OnPayloadDropped;
// these two setters actually update the queues themselves
MaxPayloadSize = options.MaxPayloadSize;
MaxPendingPayloads = options.MaxPendingPayloads;
MetricsNamePrefix = options.MetricsNamePrefix ?? "";
if (MetricsNamePrefix != "" && !BosunValidation.IsValidMetricName(MetricsNamePrefix))
{
throw new Exception("\"" + MetricsNamePrefix + "\" is not a valid metric name prefix.");
}
GetBosunUrl = options.GetBosunUrl;
BosunUrl = GetBosunUrl == null ? options.BosunUrl : GetBosunUrl();
_accessToken = options.AccessToken;
_getAccessToken = options.GetAccessToken;
ThrowOnPostFail = options.ThrowOnPostFail;
ThrowOnQueueFull = options.ThrowOnQueueFull;
ReportingInterval = options.ReportingInterval;
EnableExternalCounters = options.EnableExternalCounters;
PropertyToTagName = options.PropertyToTagName;
TagValueConverter = options.TagValueConverter;
DefaultTags = ValidateDefaultTags(options.DefaultTags);
// start continuous queue-flushing
_flushTimer = new Timer(Flush, true, 1000, 1000);
// start reporting timer
var interval = TimeSpan.FromSeconds(ReportingInterval);
_reportingTimer = new Timer(Snapshot, true, interval, interval);
// metadata timer - wait 30 seconds to start (so there is some time for metrics to be delcared)
if (options.MetaDataReportingInterval > 0)
{
_metaDataTimer = new Timer(PostMetaData, true, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(options.MetaDataReportingInterval));
}
}
public void ConstructionTest()
{
using (var logger = new RollbarLogger())
{
RollbarClient client = new RollbarClient(logger);
PayloadQueue pq = new PayloadQueue(logger, client);
Assert.IsNotNull(pq.Logger);
Assert.AreSame(logger, pq.Logger);
Assert.AreSame(client, pq.Client);
Assert.AreSame(client.Config, pq.Client.Config);
Assert.AreSame(client.Config, logger.Config);
}
}
/*The PipeMessage method is what will connect to the IoT Edge Hub and will queue the shelf data
* for processing.
*
* It will be registered as an inputMessageHandler with the IoT Edge Hub in the ExecuteAsync method
* that follows.
*/
public async Task <MessageResponse> PipeMessage(Message message, object userContext)
{
var moduleClient = userContext as ModuleClient;
if (moduleClient == null)
{
throw new InvalidOperationException("UserContext doesn't contain " + "expected values");
}
/*This section receives the data from the IoT Edge Hub and converts it to a Shelf object
* for processing through signalR to the WebApp module.
*/
byte[] messageBytes = message.GetBytes();
string messageString = Encoding.UTF8.GetString(messageBytes);
if (!string.IsNullOrEmpty(messageString))
{
try
{
_logger.LogInformation($"Receiving Message: {messageString.TrimStart('"').TrimEnd('"').Replace('\\', ' ')}");
Payload productData = JsonConvert.DeserializeObject <Payload>
(messageString.TrimStart('"').TrimEnd('"').Replace("\\", String.Empty));
PayloadQueue.QueuePayload(productData);
if (PayloadQueue.Count() > 1)
{
while (PayloadQueue.Count() > 1)
{
await PayloadQueue.DequeueAsync(new CancellationToken()); //throw away result
_logger.LogInformation("Dequeue extra data");
}
}
// catch and swallow exceptions
}
catch (AggregateException ex)
{
_logger.LogError($"Error processing message: {ex.Flatten()}");
}
catch (Exception ex)
{
_logger.LogError($"Error processing message: {ex}");
}
}
return(MessageResponse.Completed);
}
public WebhookReceiverTests()
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.Build();
var payloadQueueSettings = configuration.GetSection("PayloadQueueSettings").Get <PayloadQueueSettings>();
_payloadQueue = new PayloadQueue(payloadQueueSettings);
_payloadQueue.Clear();
var server = new TestServer(new WebHostBuilder().UseConfiguration(configuration).UseStartup <Startup>());
_client = server.CreateClient();
}
/// <summary>
/// Instantiates a new collector (the primary class of BosunReporter). You should typically only instantiate one collector for the lifetime of your
/// application. It will manage the serialization of metrics and sending data to Bosun.
/// </summary>
/// <param name="options"></param>
public MetricsCollector(BosunOptions options)
{
ExceptionHandler = options.ExceptionHandler;
if (options.SnapshotInterval < TimeSpan.FromSeconds(1))
{
throw new InvalidOperationException("options.SnapshotInterval cannot be less than one second");
}
_localMetricsQueue = new PayloadQueue(QueueType.Local);
_externalCounterQueue = new PayloadQueue(QueueType.ExternalCounters);
_localMetricsQueue.PayloadDropped += OnPayloadDropped;
_externalCounterQueue.PayloadDropped += OnPayloadDropped;
// these two setters actually update the queues themselves
MaxPayloadSize = options.MaxPayloadSize;
MaxPendingPayloads = options.MaxPendingPayloads;
MetricsNamePrefix = options.MetricsNamePrefix ?? "";
if (MetricsNamePrefix != "" && !BosunValidation.IsValidMetricName(MetricsNamePrefix))
{
throw new Exception("\"" + MetricsNamePrefix + "\" is not a valid metric name prefix.");
}
_getUrlDynamic = options.GetBosunUrl;
_fixedUrl = options.BosunUrl;
_accessToken = options.AccessToken;
_getAccessToken = options.GetAccessToken;
ThrowOnPostFail = options.ThrowOnPostFail;
ThrowOnQueueFull = options.ThrowOnQueueFull;
ReportingInterval = options.SnapshotInterval;
EnableExternalCounters = options.EnableExternalCounters;
PropertyToTagName = options.PropertyToTagName;
TagValueConverter = options.TagValueConverter;
DefaultTags = ValidateDefaultTags(options.DefaultTags);
// start continuous queue-flushing
_flushTimer = new Timer(Flush, true, 1000, 1000);
// start reporting timer
_reportingTimer = new Timer(Snapshot, true, ReportingInterval, ReportingInterval);
}
public bool TryFilterNewPayload(T payload)
{
if (PayloadQueue.Contains(payload))
{
return(false);
}
else if (PayloadQueue.Count < MaxQueueItems)
{
PayloadQueue.Enqueue(payload);
return(true);
}
else
{
PayloadQueue.TryDequeue(out _);
PayloadQueue.Enqueue(payload);
return(true);
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
/*hubContext allows ClientNotifier to act as a part of the signalR hub and send messages. See this
* for more information: https://docs.microsoft.com/en-us/aspnet/core/signalr/hubcontext?view=aspnetcore-2.1 */
var notifytask = Task.Run(async() =>
{
while (!stoppingToken.IsCancellationRequested)
{
if (PayloadQueue.Count() > 0)
{
await this.HubContext.Clients.All.SendAsync("NewData");
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
}, stoppingToken);
await Task.WhenAll(notifytask);
}
void FlushPayloadQueue(PayloadQueue queue, Uri url)
{
if (queue.PendingPayloadsCount == 0)
{
return;
}
if (!ShutdownCalled && queue.SuspendFlushingUntil > DateTime.UtcNow)
{
return;
}
while (queue.PendingPayloadsCount > 0)
{
if (!FlushPayload(url, queue))
{
break;
}
}
}
public async Task <bool> TryFilterNewPayload(T payload, Func <T, string> keyExpression)
{
var payloadKey = $"{typeof(T).Name}:{keyExpression(payload)}";
using (await _payloadLock.WaitAsync(payloadKey))
{
if (PayloadQueue.Contains(payload))
{
return(false);
}
else if (PayloadQueue.Count < MaxQueueItems)
{
PayloadQueue.Enqueue(payload);
return(true);
}
else
{
PayloadQueue.TryDequeue(out _);
PayloadQueue.Enqueue(payload);
return(true);
}
}
}
public void AddPayload(int blockNumber, ByteString payloadItemToAdd)
{
PayloadQueue.Enqueue(new PayloadItem(blockNumber, payloadItemToAdd));
}
