/// <summary>
/// Sends the events asynchronous.
/// </summary>
/// <param name="events">The events.</param>
/// <param name="transmissionSequenceNumber">The transmission sequence number.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns></returns>
public Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (events == null || !events.Any())
{
return(CompletedTask);
}
try
{
foreach (var evt in events)
{
if (cancellationToken.IsCancellationRequested)
{
return(CompletedTask);
}
var level = evt.Level.ToString();
RIExtendedMessageProperty.AttachToRequest(TraceTag, "Provider", evt.ProviderName);
RIExtendedMessageProperty.AttachToRequest(TraceTag, "Level", level);
evt.Payload.TryGetValue("Message", out object message);
_reflectInsight.SendJSON($"[{level}] {(message as string) ?? evt.ProviderName}", evt);
}
_healthReporter.ReportHealthy();
return(CompletedTask);
}
catch (Exception ex)
{
_healthReporter.ReportProblem($"{TraceTag}: Fail to send events in batch. Error details: {ex.ToString()}");
throw;
}
}
private ElasticConnectionPoolType GetEsConnectionPoolType(IHealthReporter healthReporter)
{
if (ConnectionPoolType != default(string) &&
Enum.TryParse <ElasticConnectionPoolType>(ConnectionPoolType, out var elasticConnectionPool))
{
return(elasticConnectionPool);
}
var infoMessage = $"{nameof(ElasticSearchOutput)}: Using default {DefaultConnectionPoolType} connection type.";
healthReporter.ReportHealthy(infoMessage, EventFlowContextIdentifiers.Configuration);
return(DefaultConnectionPoolType);
}
private void Initialize(BigQueryOutputConfiguration bqOutputConfiguration)
{
Debug.Assert(bqOutputConfiguration != null);
Debug.Assert(this.healthReporter != null);
if (String.IsNullOrEmpty(bqOutputConfiguration.ProjectId))
{
throw new ArgumentException("ProjectId");
}
if (String.IsNullOrEmpty(bqOutputConfiguration.DatasetId))
{
throw new ArgumentException("DatasetId");
}
if (String.IsNullOrEmpty(bqOutputConfiguration.TableId))
{
throw new ArgumentException("TableId");
}
this.Config = bqOutputConfiguration;
// Load table schema file
if (Config.TableSchemaFile != null)
{
TableSchema = LoadTableSchema(Config.TableSchemaFile);
}
if (TableSchema == null)
{
throw new Exception("table schema not set");
}
// Expand table id 1st time within force mode
ExpandTableIdIfNecessary(force: true);
// configure finished
healthReporter.ReportHealthy("TableId: " + TableIdExpanded);
var scopes = new[]
{
BigqueryService.Scope.Bigquery,
BigqueryService.Scope.BigqueryInsertdata,
BigqueryService.Scope.CloudPlatform,
BigqueryService.Scope.DevstorageFullControl
};
var credential = GoogleCredential.GetApplicationDefault()
.CreateScoped(scopes);
_BQSvc = new BigqueryService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
});
_BackOff = new ExponentialBackOff();
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
try
{
await splunkHttpEventCollectorClient.SendEventsAsync(events, cancellationToken);
healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = $"{nameof(SplunkOutput)}: An error occurred while sending data to Splunk. Exception: {e}";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Use a "Supressed" transaction scope to not be affected by callers rolling backs
using (var scope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
{
try
{
using (SqlConnection connection = new SqlConnection(ConnectionSettings.ConnectionString))
{
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
using (SqlCommand command = CreateCommand(connection))
{
var transactionName = String.Concat(nameof(SqlTableOutput), transmissionSequenceNumber);
using (command.Transaction = connection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted, transactionName))
{
try
{
var inserted = 0;
command.Prepare();
#if DEBUG
Console.WriteLine($"{nameof(SqlTableOutput)}: will try to insert #{events.Count} events into SQL Server database");
#endif
// Execute insert for each row in table
foreach (var row in CreateEventsDataTable(events))
{
foreach (var col in row)
{
command.Parameters[String.Concat(PARM_PREFIX, col.Key)].Value = col.Value ?? DBNull.Value;
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
var result = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
command.Transaction.Save(transactionName);
inserted += result;
}
command.Transaction.Commit();
_healthReporter.ReportHealthy($"{nameof(SqlTableOutput)}: #{inserted} events written to SQL Server database", EventFlowContextIdentifiers.Output);
#if DEBUG
Console.WriteLine($"{nameof(SqlTableOutput)}: #{inserted} events written to SQL Server database");
#endif
}
catch (Exception sqlex)
{
_healthReporter.ReportWarning($"{nameof(SqlTableOutput)}: error writing data to SQL database. {sqlex.Message}", EventFlowContextIdentifiers.Output);
#if DEBUG
Console.WriteLine($"{nameof(SqlTableOutput)}: error writing data to SQL database. {sqlex.Message}");
Console.WriteLine(sqlex.StackTrace);
#endif
try
{
command.Transaction.Rollback(transactionName);
connection.Close();
}
catch (Exception rex)
{
_healthReporter.ReportWarning($"{nameof(SqlTableOutput)}: error rolling back. {rex.Message}", EventFlowContextIdentifiers.Output);
#if DEBUG
Console.WriteLine($"{nameof(SqlTableOutput)}: error rolling back. {rex.Message}");
Console.WriteLine(rex.StackTrace);
#endif
}
throw;
}
}
}
connection.Close();
}
// Not really needed, since the scope is "Suppressed"
scope.Complete();
}
catch (Exception ex)
{
_healthReporter.ReportProblem($"{nameof(SqlTableOutput)}: write of events data to SQL Server database has failed. {ex.Message}", EventFlowContextIdentifiers.Output);
#if DEBUG
Console.WriteLine($"{nameof(SqlTableOutput)}: write of events data to SQL Server database has failed. {ex.Message}");
Console.WriteLine(ex.StackTrace);
#endif
}
}
}