/// <summary>
/// Read data and transmit via HTTP to Splunk
/// </summary>
/// <param name="query"></param>
/// <param name="sqlConnectionObject"></param>
private static void ReadAndTransmitData(string query, SqlConnection sqlConnectionObject)
{
DataTable dataTable = new DataTable();
string kvpValue = "";
var localisExecutingSQLCommand = false;
try
{
if (sqlConnectionObject.State == ConnectionState.Open)
{
if (string.IsNullOrEmpty(query))
{
throw new Exception("Query string is null or empty");
}
lock (_updateisExecutingSQLCommand)
{
localisExecutingSQLCommand = _isExecutingSQLCommand;
}
log.DebugFormat("localisExecutingSQLCommand = {0}", localisExecutingSQLCommand);
if (!localisExecutingSQLCommand)
{
SqlCommand command = new SqlCommand(query, sqlConnectionObject);
lock (_updateisExecutingSQLCommand)
{
_isExecutingSQLCommand = true;
}
dataTable.Load(command.ExecuteReader());
lock (_updateisExecutingSQLCommand)
{
_isExecutingSQLCommand = false;
}
log.InfoFormat("{0} rows retrieved", dataTable.Rows.Count);
if (dataTable.Rows.Count > 0)
{
//Build the additional KVP values to Append
var additionalKVPValues = new StringBuilder();
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost);
additionalKVPValues.AppendFormat("{0}=\"{1}\" ", "SourceData", RuntimeOptions.SplunkSourceData);
//Get the KVP string for the records
kvpValue = dataTable.ToKVP(additionalKVPValues.ToString(), RuntimeOptions.SQLTimestampField, RuntimeOptions.SplunkEventTimestampFormat);
log.DebugFormat("KVP Values");
log.DebugFormat("{0}", kvpValue);
//Transmit the records
var result = SplunkHTTPClient.TransmitValues(kvpValue);
log.DebugFormat("Transmit Values Result - {0}", result);
//If successful then write the last sequence value to disk
if (result.StatusCode == HttpStatusCode.OK)
{
log.DebugFormat("Writing Cache File");
// Write the last sequence value to the cache value named for the SQLSequence Field. Order the result set by the sequence field then select the first record
WriteCacheFile(dataTable, CacheFilename, RuntimeOptions);
if (ReadTimer.Interval != RuntimeOptions.ReadInterval)
{
//Reset timer interval
ClearTimerBackoff(ReadTimer, RuntimeOptions);
}
}
else
{
// Implement a timer backoff so we don't flood the endpoint
IncrementTimerBackoff(ReadTimer, RuntimeOptions);
log.WarnFormat("HTTP Transmission not OK - {0}", result);
}
}
}
else
{
log.DebugFormat("SQL command already executing. Skipping this cycle.");
}
}
else
{
log.Warn("SQL Connection not open");
}
}
catch (Exception ex)
{
log.Error(ex);
}
finally
{
dataTable.Dispose();
}
}
/// <summary>
/// Read data and transmit via HTTP to Splunk
/// </summary>
/// <param name="query"></param>
/// <param name="sqlConnectionObject"></param>
private static void ReadAndTransmitData(aaLogReader.aaLogReader aaLogReader)
{
string kvpValue = "";
List <LogRecord> logRecords = new List <LogRecord>();
List <LogRecord> lgxRecords = new List <LogRecord>();
try
{
//Get the last message number written
var lastMessageNumberWritten = LastMessageNumberWritten;
if (lastMessageNumberWritten == 0)
{
log.DebugFormat("Executing GetUnreadRecords for {0} records", runtimeOptions.MaxRecords);
// Get records starting with last log record and move backwards since we are working with no existing cache file
//AALogReader.Options.IgnoreCacheFileOnFirstRead = true;
//logRecords = AALogReader.GetRecordsByEndTimestampAndCount(System.DateTime.Now.AddMinutes(1), (int)runtimeOptions.MaxRecords);
logRecords = AALogReader.GetUnreadRecords((ulong)runtimeOptions.MaxRecords);
}
else
{
log.DebugFormat("Executing GetRecordsByStartMessageNumberAndCount for message number {0} with a maximum of {1} records", lastMessageNumberWritten + 1, runtimeOptions.MaxRecords);
// Get records based on last cached message number
logRecords = AALogReader.GetRecordsByStartMessageNumberAndCount(lastMessageNumberWritten + 1, (int)runtimeOptions.MaxRecords);
}
log.InfoFormat("{0} records retrieved", logRecords.Count);
if (logRecords.Count > 0)
{
//Build the additional KVP values to Append
var additionalKVPValues = new StringBuilder();
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost);
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData);
//Get the KVP string for the records
kvpValue = logRecords.ToKVP(additionalKVPValues.ToString());
//Transmit the records
var result = SplunkHTTPClient.TransmitValues(kvpValue);
log.DebugFormat("Transmit Values Result - {0}", result);
//If successful then write the last sequence value to disk
if (result.StatusCode == HttpStatusCode.OK)
{
log.DebugFormat("Writing Cache File");
// Write the last sequence value to the cache value named for the SQLSequence Field. Order the result set by the sequence field then select the first record
WriteCacheFile(logRecords, CacheFilename, RuntimeOptions);
if (ReadTimer.Interval != RuntimeOptions.ReadInterval)
{
//Reset timer interval
ClearTimerBackoff(ReadTimer, RuntimeOptions);
}
}
else
{
// Implement a timer backoff so we don't flood the endpoint
IncrementTimerBackoff(ReadTimer, RuntimeOptions);
log.WarnFormat("HTTP Transmission not OK - {0}", result);
}
}
var aaLGXDirectory = runtimeOptions.AALGXDirectory ?? "";
// Parse AALGX Files
if (aaLGXDirectory != "")
{
log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory);
if (Directory.Exists(aaLGXDirectory))
{
var successFolder = aaLGXDirectory + "\\success";
if (!Directory.Exists(successFolder))
{
Directory.CreateDirectory(successFolder);
}
var errorFolder = aaLGXDirectory + "\\error";
if (!Directory.Exists(errorFolder))
{
Directory.CreateDirectory(errorFolder);
}
string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx");
foreach (string fileName in filesList)
{
log.InfoFormat("Processing file {0}", fileName);
var aaLGXRecords = aaLgxReader.ReadLogRecords(fileName);
log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName);
if (aaLGXRecords.Count() > 0)
{
//Build the additional KVP values to Append
var additionalKVPValues = new StringBuilder();
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost);
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData);
//Get the KVP string for the records
kvpValue = aaLGXRecords.ToKVP(additionalKVPValues.ToString());
//Transmit the records
var result = SplunkHTTPClient.TransmitValues(kvpValue);
log.DebugFormat("Transmit Values Result - {0}", result);
//If successful then write the last sequence value to disk
if (result.StatusCode == HttpStatusCode.OK)
{
log.InfoFormat("Successfully transmitted {0} records from file {1}", aaLGXRecords.Count, fileName);
log.InfoFormat("Moving {0} to {1}", fileName, successFolder);
var destinationFilename = Path.Combine(successFolder, Path.GetFileName(fileName));
if (File.Exists(destinationFilename))
{
log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName);
try
{
File.Delete(destinationFilename);
}
catch (Exception ex)
{
log.Error(ex);
}
}
try
{
File.Move(fileName, destinationFilename);
}
catch
{
log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename);
}
if (ReadTimer.Interval != RuntimeOptions.ReadInterval)
{
//Reset timer interval
ClearTimerBackoff(ReadTimer, RuntimeOptions);
}
}
else
{
// Implement a timer backoff so we don't flood the endpoint
IncrementTimerBackoff(ReadTimer, RuntimeOptions);
log.WarnFormat("HTTP Transmission not OK - {0}", result);
log.InfoFormat("Moving {0} to {1}", fileName, errorFolder);
var destinationFilename = Path.Combine(errorFolder, Path.GetFileName(fileName));
if (File.Exists(destinationFilename))
{
log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName);
try
{
File.Delete(destinationFilename);
}
catch (Exception ex)
{
log.Error(ex);
}
}
try
{
File.Move(fileName, destinationFilename);
}
catch
{
log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename);
}
}
}
}
}
else
{
log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory);
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
