/// <summary>
/// Try to send the given message. On failure, reset and try again.
/// If that fails, just rethrow the exception.
/// </summary>
/// <param name="message">The message to be sent to the Wavefront proxy.</param>
public async Task WriteAsync(string message)
{
var bytes = Encoding.UTF8.GetBytes(message);
try
{
// Might be NPE due to previously failed call to ResetSocket.
await socketOutputStream.WriteAsync(bytes, 0, bytes.Length);
writeSuccesses.Inc();
}
catch (Exception e)
{
try
{
logger.LogWarning(0, e, "Attempting to reset socket connection.");
await ResetSocketAsync();
await socketOutputStream.WriteAsync(bytes, 0, bytes.Length);
writeSuccesses.Inc();
}
catch (Exception e2)
{
writeErrors.Inc();
throw new IOException(e2.Message, e2);
}
}
}
/// <see cref="IWavefrontTracingSpanSender.SendSpan"/>
public void SendSpan(string name, long startMillis, long durationMillis, string source,
Guid traceId, Guid spanId, IList <Guid> parents,
IList <Guid> followsFrom, IList <KeyValuePair <string, string> > tags,
IList <SpanLog> spanLogs)
{
if (tracingProxyConnectionHandler == null)
{
spansDiscarded.Inc();
if (spanLogs != null && spanLogs.Count > 0)
{
spanLogsDiscarded.Inc();
}
logger.LogWarning("Can't send data to Wavefront. " +
"Please configure histogram distribution port for Wavefront proxy.");
return;
}
string lineData;
try
{
lineData = Utils.TracingSpanToLineData(name, startMillis,
durationMillis, source, traceId, spanId, parents, followsFrom, tags, spanLogs,
defaultSource);
spansValid.Inc();
}
catch (ArgumentException e)
{
spansInvalid.Inc();
throw e;
}
_ = tracingProxyConnectionHandler.SendDataAsync(lineData).ContinueWith(
task =>
{
if (task.Exception == null)
{
// Send valid span logs only if the span was successfully sent
if (spanLogs != null && spanLogs.Count > 0)
{
SendSpanLogs(traceId, spanId, spanLogs, lineData);
}
}
else
{
spansDropped.Inc();
if (spanLogs != null && spanLogs.Count > 0)
{
spanLogsDropped.Inc();
}
tracingProxyConnectionHandler.IncrementFailureCount();
}
});
}
/// <see cref="IWavefrontHistogramSender.SendDistribution"/>
public void SendDistribution(string name, IList <KeyValuePair <double, int> > centroids,
ISet <HistogramGranularity> histogramGranularities,
long?timestamp, string source,
IDictionary <string, string> tags)
{
if (histogramProxyConnectionHandler == null)
{
histogramsDiscarded.Inc();
logger.LogWarning("Can't send data to Wavefront. " +
"Please configure histogram distribution port for Wavefront proxy.");
return;
}
string lineData;
try
{
lineData = Utils.HistogramToLineData(name, centroids, histogramGranularities,
timestamp, source, tags, defaultSource);
histogramsValid.Inc();
}
catch (ArgumentException e)
{
histogramsInvalid.Inc();
throw e;
}
_ = histogramProxyConnectionHandler.SendDataAsync(lineData).ContinueWith(
task =>
{
histogramsDropped.Inc();
histogramProxyConnectionHandler.IncrementFailureCount();
}, TaskContinuationOptions.OnlyOnFaulted);
}
/// <see cref="Entities.Metrics.IWavefrontMetricSender.SendMetric"/>
public void SendMetric(string name, double value, long?timestamp, string source,
IDictionary <string, string> tags)
{
if (metricsProxyConnectionHandler == null)
{
pointsDiscarded.Inc();
logger.LogWarning("Can't send data to Wavefront. " +
"Please configure metrics port for Wavefront proxy.");
return;
}
string lineData;
try
{
lineData = Utils.MetricToLineData(name, value, timestamp, source, tags,
defaultSource);
pointsValid.Inc();
}
catch (ArgumentException e)
{
pointsInvalid.Inc();
throw e;
}
_ = metricsProxyConnectionHandler.SendDataAsync(lineData).ContinueWith(
task =>
{
pointsDropped.Inc();
metricsProxyConnectionHandler.IncrementFailureCount();
}, TaskContinuationOptions.OnlyOnFaulted);
}
private void Send(WavefrontSpan span)
{
try
{
var context = (WavefrontSpanContext)span.Context;
var parentReferences = span.GetParents();
var followReferences = span.GetFollows();
var parents = parentReferences?
.Select(parent => parent.SpanContext.GetSpanId())
.ToList();
var follows = followReferences?
.Select(follow => follow.SpanContext.GetSpanId())
.ToList();
WavefrontSender.SendSpan(
span.GetOperationName(), span.GetStartTimeMicros() / 1000,
span.GetDurationMicros() / 1000, Source, context.GetTraceId(),
context.GetSpanId(), parents, follows, span.GetTagsAsList().ToList(),
reportSpanLogs ? span.GetSpanLogs().ToList() : null
);
}
catch (IOException e)
{
if (LoggingAllowed())
{
logger.LogWarning(0, e, "Error reporting span: " + span);
}
spansDropped?.Inc();
reportErrors?.Inc();
}
}
/// <inheritdoc />
public void Report(WavefrontSpan span)
{
spansReceived?.Inc();
if (!spanBuffer.TryAdd(span))
{
spansDropped?.Inc();
if (LoggingAllowed())
{
logger.LogWarning("Buffer full, dropping span: " + span);
if (spansDropped != null)
{
logger.LogWarning("Total spans dropped: " + spansDropped.Count);
}
}
}
}
/// <see cref="IWavefrontHistogramSender.SendDistribution"/>
public void SendDistribution(string name,
IList <KeyValuePair <double, int> > centroids,
ISet <HistogramGranularity> histogramGranularities,
long?timestamp,
string source,
IDictionary <string, string> tags)
{
string lineData;
try
{
lineData = Utils.HistogramToLineData(name, centroids, histogramGranularities,
timestamp, source, tags, DefaultSource);
histogramsValid.Inc();
}
catch (ArgumentException e)
{
histogramsInvalid.Inc();
throw e;
}
if (!histogramsBuffer.TryAdd(lineData))
{
histogramsDropped.Inc();
logger.LogWarning("Buffer full, dropping histograms: " + lineData);
}
}
/// <see cref="IWavefrontTracingSpanSender.SendSpan"/>
public void SendSpan(string name, long startMillis, long durationMillis, string source,
Guid traceId, Guid spanId, IList <Guid> parents,
IList <Guid> followsFrom, IList <KeyValuePair <string, string> > tags,
IList <SpanLog> spanLogs)
{
string lineData;
try
{
lineData = Utils.TracingSpanToLineData(name, startMillis,
durationMillis, source, traceId, spanId, parents, followsFrom, tags, spanLogs,
DefaultSource);
spansValid.Inc();
}
catch (ArgumentException e)
{
spansInvalid.Inc();
throw e;
}
if (tracingSpansBuffer.TryAdd(lineData))
{
// Enqueue span logs for sending only if the span was successfully enqueued
if (spanLogs != null && spanLogs.Count > 0)
{
SendSpanLogs(traceId, spanId, spanLogs, lineData);
}
}
else
{
spansDropped.Inc();
if (spanLogs != null && spanLogs.Count > 0)
{
spanLogsDropped.Inc();
}
logger.LogWarning("Buffer full, dropping span: " + lineData);
}
}
/// <summary>
/// Flushes the outputStream best-effort. If that fails, we reset the connection.
/// </summary>
public async Task FlushAsync()
{
try
{
await socketOutputStream.FlushAsync();
flushSuccesses.Inc();
}
catch (Exception e)
{
flushErrors.Inc();
logger.LogWarning(0, e, "Attempting to reset socket connection.");
await ResetSocketAsync();
}
}
public void Connect()
{
if (reconnectingSocket != null)
{
throw new InvalidOperationException("Already connected");
}
try
{
reconnectingSocket = new ReconnectingSocket(host, port, sdkMetricsRegistry,
entityPrefix + "socket", loggerFactory);
}
catch (Exception e)
{
connectErrors.Inc();
throw new IOException(e.Message, e);
}
}
private void SendSpanLogs(Guid traceId, Guid spanId, IList <SpanLog> spanLogs, string span)
{
try
{
string lineData = Utils.SpanLogsToLineData(traceId, spanId, spanLogs, span);
spanLogsValid.Inc();
if (!spanLogsBuffer.TryAdd(lineData))
{
spanLogsDropped.Inc();
logger.LogWarning("Buffer full, dropping span logs: " + lineData);
}
}
catch (Exception)
{
spanLogsInvalid.Inc();
logger.LogWarning($"Unable to serialize span logs to json: traceId:{traceId}" +
$" spanId:{spanId} spanLogs:{spanLogs}");
}
}
/// <summary>
/// Flushes the current metrics snapshot in Wavefront data format.
/// </summary>
/// <param name="metricsData">The current snapshot of metrics.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>True if metrics were successfully flushed, false otherwise.</returns>
public async Task <bool> FlushAsync(
MetricsDataValueSource metricsData,
CancellationToken cancellationToken = default)
{
Logger.Trace("Flushing metrics snapshot");
try
{
await WriteAsync(metricsData, cancellationToken);
}
catch (Exception e)
{
reporterErrors.Inc();
Logger.Error(e.Message);
return(false);
}
Logger.Trace("Flushed metrics snapshot");
return(true);
}
private void SendSpanLogs(Guid traceId, Guid spanId, IList <SpanLog> spanLogs, string span)
{
try
{
string lineData = Utils.SpanLogsToLineData(traceId, spanId, spanLogs, span);
spanLogsValid.Inc();
_ = tracingProxyConnectionHandler.SendDataAsync(lineData).ContinueWith(
task =>
{
spanLogsDropped.Inc();
tracingProxyConnectionHandler.IncrementFailureCount();
}, TaskContinuationOptions.OnlyOnFaulted);
}
catch (Exception)
{
spanLogsInvalid.Inc();
logger.LogWarning($"Unable to serialize span logs to json: traceId:{traceId}" +
$" spanId:{spanId} spanLogs:{spanLogs}");
}
}
/// <see cref="Entities.Metrics.IWavefrontMetricSender.SendMetric"/>
public void SendMetric(string name, double value, long?timestamp, string source,
IDictionary <string, string> tags)
{
string lineData;
try
{
lineData = Utils.MetricToLineData(name, value, timestamp, source, tags,
DefaultSource);
pointsValid.Inc();
}
catch (ArgumentException e)
{
pointsInvalid.Inc();
throw e;
}
if (!metricsBuffer.TryAdd(lineData))
{
pointsDropped.Inc();
logger.LogWarning("Buffer full, dropping metric point: " + lineData);
}
}
/// <inheritdoc />
public void Write(string context, string name, IEnumerable <string> columns,
IEnumerable <object> values, MetricTags tags, DateTime timestamp)
{
// Do not report App Metrics' internal metrics (e.g., report_success counter)
if (context == InternalMetricsContext)
{
return;
}
try
{
string metricTypeValue =
tags.Values[Array.IndexOf(tags.Keys, Pack.MetricTagsTypeKey)];
var data = columns.Zip(values, (column, value) => new { column, value })
.ToDictionary(pair => pair.column, pair => pair.value);
if (metricTypeValue == Pack.ApdexMetricTypeValue)
{
WriteApdex(context, name, data, tags, timestamp);
apdexesReported.Inc();
}
else if (metricTypeValue == Pack.CounterMetricTypeValue)
{
WriteCounter(context, name, data, tags, timestamp);
if (DeltaCounterOptions.IsDeltaCounter(tags))
{
deltaCountersReported.Inc();
}
else
{
countersReported.Inc();
}
}
else if (metricTypeValue == Pack.GaugeMetricTypeValue)
{
WriteGauge(context, name, data, tags, timestamp);
gaugesReported.Inc();
}
else if (metricTypeValue == Pack.HistogramMetricTypeValue)
{
WriteHistogram(context, name, data, tags, timestamp);
if (WavefrontHistogramOptions.IsWavefrontHistogram(tags))
{
wfHistogramsReported.Inc();
}
else
{
histogramsReported.Inc();
}
}
else if (metricTypeValue == Pack.MeterMetricTypeValue)
{
WriteMeter(context, name, data, tags, timestamp);
metersReported.Inc();
}
else if (metricTypeValue == Pack.TimerMetricTypeValue)
{
WriteMeter(context, name, data, tags, timestamp);
WriteHistogram(context, name, data, tags, timestamp);
timersReported.Inc();
}
}
catch (Exception e)
{
writerErrors.Inc();
throw e;
}
}
private void InternalFlush(BlockingCollection <string> buffer, string format,
string entityPrefix, WavefrontSdkDeltaCounter dropped, WavefrontSdkDeltaCounter reportErrors)
{
var batch = GetBatch(buffer);
if (batch.Count == 0)
{
return;
}
try
{
using (var stream = BatchToStream(batch))
{
int statusCode = directService.Report(format, stream);
sdkMetricsRegistry.DeltaCounter(entityPrefix + ".report." + statusCode).Inc();
if ((statusCode >= 400 && statusCode < 600) || statusCode == Constants.HttpNoResponse)
{
switch (statusCode)
{
case 401:
logger.LogWarning("Error sending " + entityPrefix + " to Wavefront (HTTP " + statusCode + "). " +
"Please verify that your API Token is correct! All " + entityPrefix + " are " +
"discarded.");
dropped.Inc(batch.Count);
break;
case 403:
if (format.Equals(Constants.WavefrontMetricFormat))
{
logger.LogWarning("Error sending " + entityPrefix + " to Wavefront (HTTP " + statusCode + "). " +
"Please verify that Direct Data Ingestion is enabled for your account! " +
"All " + entityPrefix + " are discarded.");
}
else
{
logger.LogWarning("Error sending " + entityPrefix + " to Wavefront (HTTP " + statusCode + "). " +
"Please verify that Direct Data Ingestion and " + entityPrefix + " are " +
"enabled for your account! All " + entityPrefix + " are discarded.");
}
dropped.Inc(batch.Count);
break;
default:
logger.LogWarning("Error sending " + entityPrefix + " to Wavefront (HTTP " + statusCode + "). Data " +
"will be requeued and resent.");
int numAddedBackToBuffer = 0;
foreach (var item in batch)
{
if (buffer.TryAdd(item))
{
numAddedBackToBuffer++;
}
else
{
int numDropped = batch.Count - numAddedBackToBuffer;
dropped.Inc(numDropped);
logger.LogWarning("Buffer full, dropping " + numDropped + " " + entityPrefix + ". Consider increasing " +
"the batch size of your sender to increase throughput.");
break;
}
}
break;
}
}
}
}
catch (IOException e) {
dropped.Inc(batch.Count);
reportErrors.Inc();
throw e;
}
}
private async Task ConnectAsync(bool isReset)
{
if (!await connectSemaphore.WaitAsync(TimeSpan.Zero))
{
// skip if another thread is already attempting to connect
return;
}
try
{
if (isReset)
{
// Close the outputStream and connection
try
{
socketOutputStream?.Close();
}
catch (IOException e)
{
logger.LogInformation(0, e, "Could not flush and close socket.");
}
client.Close();
client = new TcpClient
{
ReceiveTimeout = serverReadTimeoutMillis
};
}
// Open a connection and instantiate the outputStream
try
{
var connectTask = client.ConnectAsync(host, port);
var timeoutTask = Task.Delay(serverConnectTimeoutMillis);
var completedTask = await Task.WhenAny(connectTask, timeoutTask);
if (completedTask == connectTask)
{
socketOutputStream =
Stream.Synchronized(new BufferedStream(client.GetStream(), bufferSize));
if (isReset)
{
resetSuccesses.Inc();
}
logger.LogInformation(
string.Format("Successfully connected to {0}:{1}", host, port));
}
else
{
logger.LogWarning(string.Format("Unable to connect to {0}:{1}", host, port));
client.Close();
}
}
catch (Exception e)
{
if (isReset)
{
resetErrors.Inc();
}
logger.LogWarning(0, e,
string.Format("Unable to connect to {0}:{1}", host, port));
client.Close();
throw new IOException(e.Message, e);
}
}
finally
{
connectSemaphore.Release();
}
}
/// <summary>
/// Increments the failure count by one.
/// </summary>
public void IncrementFailureCount()
{
errors.Inc();
}