internal string GetDeleteSyntax(IInfluxRetentionPolicy rp = null, IList <string> whereClause = null)
{
if (!String.IsNullOrWhiteSpace(Name) && !String.IsNullOrWhiteSpace(Name))
{
string whereClauseText = "";
if (whereClause != null)
{
whereClauseText = $" where { String.Join(" and ", whereClause?.ToArray())}";
}
if (rp != null)
{
return($"DELETE from \"{rp.Name}\".\"{Name}\" {whereClauseText}");
}
else
{
return($"DELETE from \"{Name}\" {whereClauseText}");
}
}
else if (String.IsNullOrWhiteSpace(Name))
{
throw new ArgumentException("Measurement name is not set");
}
return(null);
}
/// <summary>
/// Creates a retention policy
/// </summary>
/// <param name="policy">An instance of the Retention Policy, DBName, Name and Duration must be set</param>
/// <returns>True: Success</returns>
public async Task <bool> CreateRetentionPolicyAsync(IInfluxRetentionPolicy policy)
{
var query = (policy as InfluxRetentionPolicy).GetCreateSyntax();
if (query != null)
{
var response = await GetAsync(new Dictionary <string, string>() { { "q", query } });
if (response.StatusCode == HttpStatusCode.OK)
{
(policy as InfluxRetentionPolicy).Saved = true;
return(true);
}
}
return(false);
}
internal string GetDropSyntax(IInfluxRetentionPolicy rp = null)
{
if (!String.IsNullOrWhiteSpace(Name) && !String.IsNullOrWhiteSpace(Name))
{
if (rp != null)
{
return($"DROP MEASUREMENT \"{rp.Name}\".\"{Name}\"");
}
else
{
return($"DROP MEASUREMENT \"{Name}\"");
}
}
else if (String.IsNullOrWhiteSpace(Name))
{
throw new ArgumentException("Measurement name is not set");
}
return(null);
}
public async Task <ProcessStatus> ProcessGenericFile(string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus();
int failedReqCount = 0;
try
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var r = GetFileLayout(InputFileName);
if (r.ExitCode != ExitCode.Success)
{
return(r);
}
IInfluxDatabase dbStructure;
if (settings.GenericFile.Filter != Filters.None)
{
var filterColumns = new List <GenericColumn>();
if (settings.GenericFile.Filter == Filters.Columns)
{
if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
{
Logger.LogLine(LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
}
else
{
filterColumns = ParseGenericColumns(settings.GenericFile.ColumnsFilter.ToString());
}
}
dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName);
ColumnHeaders = FilterGenericColumns(ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);
}
var validity = ValidateData(InputFileName);
var failureReasons = new Dictionary <Type, FailureTracker>();
List <IInfluxDatapoint> points = new List <IInfluxDatapoint>(), retryQueue = new List <IInfluxDatapoint>();
IInfluxRetentionPolicy policy = null;
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy()
{
Name = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync(policy))
{
throw new InvalidOperationException("Unable to create retention policy");
}
}
}
else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
{
throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
}
foreach (var line in File.ReadLines(InputFileName).Skip(settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
{
if (String.IsNullOrWhiteSpace(line) || (!String.IsNullOrEmpty(settings.GenericFile.CommentMarker) && line.StartsWith(settings.GenericFile.CommentMarker)))
{
continue;
}
try
{
result.PointsFound++;
var point = ProcessGenericLine(line, ColumnHeaders);
if (point == null)
{
result.PointsFailed++;
}
else
{
point.Retention = policy;
points.Add(point);
}
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count(p => p.Saved);
}
else
{
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 3)
{
break;
}
}
//a point will be either posted to Influx or in retry queue
points.Clear();
}
}
catch (Exception e)
{
result.PointsFailed++;
var type = e.GetType();
if (e is InfluxDBException)
{
if ((e as InfluxDBException).Reason == "Partial Write")
{
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
points.Clear();
}
}
if (!failureReasons.ContainsKey(type))
{
failureReasons.Add(type, new FailureTracker()
{
ExceptionType = type, Message = e.Message
});
}
failureReasons[type].LineNumbers.Add(result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);
//avoid too many failures, may be config is wrong
if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
{
Logger.LogLine(LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
Logger.LogLine(LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
break;
}
}
if (result.PointsFailed > 0 || retryQueue.Count > 0)
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
}
else
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound);
}
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
}
}
//retry all previously failed points
if (retryQueue.Count > 0)
{
Logger.LogLine(LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count);
try
{
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear();
}
else
{
result.PointsFailed += retryQueue.Count;
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
}
}
catch (InfluxDBException e)
{
if (e.Reason == "Partial Write")
{
}
}
}
stopwatch.Stop();
if (result.PointsFailed > 0)
{
Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
foreach (var f in failureReasons.Values)
{
Logger.LogLine(LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers.Take(5)), f.ExceptionType, f.Message);
}
if (result.PointsFailed == result.PointsFound)
{
result.ExitCode = ExitCode.UnableToProcess;
}
else
{
result.ExitCode = ExitCode.ProcessedWithErrors;
}
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnableToProcess;
}
return(result);
}
/// <summary>
/// Drops a InfluxMeasurement for a given retention policy
/// </summary>
/// <param name="im">An instance of IInfluxMeasurement</param>
/// <param name="rp">An instance of IInfluxRetentionPolicy, optional</param>
/// <returns>True: Success</returns>
public async Task <bool> DropMeasurementAsync(IInfluxDatabase db, IInfluxMeasurement im, IInfluxRetentionPolicy rp = null)
{
var query = (im as InfluxMeasurement).GetDropSyntax();
if (query != null)
{
var endPoint = new Dictionary <string, string>()
{
{ "db", db.Name }, { "q", query }
};
if (rp != null)
{
endPoint.Add("rp", rp.Name);
}
var response = await GetAsync(endPoint);
if (response.StatusCode == HttpStatusCode.OK)
{
var content = await response.Content.ReadAsStringAsync();
if (content.Contains("error"))
{
throw new InfluxDBException("Drop Failed", new Regex(@"\""error\"":\""(.*?)\""").Match(content).Groups[1].Value);
}
(im as InfluxMeasurement).Deleted = true;
return(true);
}
}
return(false);
}
public async Task <ProcessStatus> ProcessPerfMonLog(string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus();
int linesProcessed = 0;
int failedLines = 0;
int failedReqCount = 0;
try
{
var failureReasons = new Dictionary <Type, FailureTracker>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var firstLine = File.ReadLines(InputFileName).FirstOrDefault();
var firstCol = firstLine.Substring(0, firstLine.IndexOf(','));
if (!firstCol.Contains("PDH-CSV"))
{
throw new InvalidDataException("Input file is not a Standard Perfmon csv file");
}
var x = Regex.Matches(firstCol, "([-0-9]+)");
if (x.Count > 0)
{
minOffset = int.Parse(x[3].ToString());
}
//get the column headers
List <PerfmonCounter> pecrfCounters;
try
{
pecrfCounters = ParsePerfMonFileHeader(firstLine);
}
catch (Exception ex)
{
throw new InvalidDataException("Unable to parse file headers", ex);
}
IInfluxDatabase dbStructure;
IEnumerable <IGrouping <string, PerfmonCounter> > perfGroup;
if (settings.PerfmonFile.Filter != Filters.None)
{
var filterColumns = ParsePerfMonFileHeader(settings.PerfmonFile.ColumnsFilter.ToString(), false);
dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName);
perfGroup = FilterPerfmonLogColumns(pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy(p => p.PerformanceObject);
}
else
{
perfGroup = pecrfCounters.GroupBy(p => p.PerformanceObject);
}
List <IInfluxDatapoint> points = null, retryQueue = new List <IInfluxDatapoint>();
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy()
{
Name = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync(policy))
{
throw new InvalidOperationException("Unable to create retention policy");
}
}
}
else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
{
throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
}
//Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
foreach (var line in File.ReadLines(InputFileName).Skip(1))
{
try
{
var linePoints = ProcessPerfmonLogLine(line, perfGroup);
linesProcessed++;
if (linePoints == null || linePoints.Count == 0)
{
failedLines++;
}
else
{
result.PointsFound += linePoints.Count;
if (points == null)
{
points = linePoints;
}
else
{
points.AddRange(linePoints);
}
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count(p => p.Saved);
}
else
{
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 4)
{
break;
}
}
//a point will be either posted to Influx or in retry queue
points = null;
}
}
}
catch (Exception e)
{
failedLines++;
var type = e.GetType();
if (!failureReasons.ContainsKey(type))
{
failureReasons.Add(type, new FailureTracker()
{
ExceptionType = type, Message = e.Message
});
}
failureReasons[type].LineNumbers.Add(linesProcessed);
}
if (failedLines > 0 || retryQueue.Count > 0)
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
}
else
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound);
}
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
}
}
if (retryQueue.Count > 0)
{
Logger.LogLine(LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count);
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear();
}
else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
else
{
result.PointsFailed += retryQueue.Count;
}
}
pecrfCounters.Clear();
stopwatch.Stop();
if (failedLines > 0 || retryQueue.Count > 0)
{
Logger.Log(LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
foreach (var f in failureReasons.Values)
{
Logger.LogLine(LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers), f.ExceptionType, f.Message);
}
if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count)
{
result.ExitCode = ExitCode.UnableToProcess;
}
else
{
result.ExitCode = ExitCode.ProcessedWithErrors;
}
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnknownError;
}
return(result);
}
/// <summary>
/// Creates a retention policy
/// </summary>
/// <param name="policy">An instance of the Retention Policy, DBName, Name and Duration must be set</param>
/// <returns>True: Success</returns>
public async Task<bool> CreateRetentionPolicyAsync (IInfluxRetentionPolicy policy)
{
var query = (policy as InfluxRetentionPolicy).GetCreateSyntax ();
if (query != null)
{
var response = await GetAsync (new Dictionary<string, string> () { { "q", query } });
if (response.StatusCode == HttpStatusCode.OK)
{
(policy as InfluxRetentionPolicy).Saved = true;
return true;
}
}
return false;
}
public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus ();
int linesProcessed = 0;
int failedLines = 0;
int failedReqCount = 0;
try
{
var failureReasons = new Dictionary<Type, FailureTracker> ();
Stopwatch stopwatch = new Stopwatch ();
stopwatch.Start ();
var firstLine = File.ReadLines (InputFileName).FirstOrDefault ();
var firstCol = firstLine.Substring (0, firstLine.IndexOf (','));
if (!firstCol.Contains ("PDH-CSV"))
throw new InvalidDataException ("Input file is not a Standard Perfmon csv file");
var x = Regex.Matches (firstCol, "([-0-9]+)");
if (x.Count > 0)
minOffset = int.Parse (x[3].ToString ());
//get the column headers
List<PerfmonCounter> pecrfCounters;
try
{
pecrfCounters = ParsePerfMonFileHeader (firstLine);
}
catch (Exception ex)
{
throw new InvalidDataException ("Unable to parse file headers", ex);
}
IInfluxDatabase dbStructure;
IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup;
if (settings.PerfmonFile.Filter != Filters.None)
{
var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false);
dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject);
}
else
{
perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject);
}
List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> ();
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy ()
{
Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync (policy))
throw new InvalidOperationException ("Unable to create retention policy");
}
}
else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
//Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
foreach (var line in File.ReadLines (InputFileName).Skip (1))
{
try
{
var linePoints = ProcessPerfmonLogLine (line, perfGroup);
linesProcessed++;
if (linePoints == null || linePoints.Count == 0)
failedLines++;
else
{
result.PointsFound += linePoints.Count;
if (points == null)
points = linePoints;
else
points.AddRange (linePoints);
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count;
}
else
{
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 4)
break;
}
//a point will be either posted to Influx or in retry queue
points = null;
}
}
}
catch (Exception e)
{
failedLines++;
var type = e.GetType ();
if (!failureReasons.ContainsKey (type))
failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
failureReasons[type].LineNumbers.Add (linesProcessed);
}
if (failedLines > 0 || retryQueue.Count > 0)
Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
else
Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound);
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear ();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
}
}
if (retryQueue.Count > 0)
{
Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear ();
}
else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
else
{
result.PointsFailed += retryQueue.Count;
}
}
pecrfCounters.Clear ();
stopwatch.Stop ();
if (failedLines > 0 || retryQueue.Count > 0)
{
Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
foreach (var f in failureReasons.Values)
Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message);
if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count)
result.ExitCode = ExitCode.UnableToProcess;
else
result.ExitCode = ExitCode.ProcessedWithErrors;
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnknownError;
}
return result;
}
