public async Task TestGetInfluxDBStructureAsync_InvalidDB() { try { var client = new InfluxDBClient(influxUrl, dbUName, dbpwd); var r = await client.GetInfluxDBStructureAsync("InvalidDB"); Assert.IsTrue(r != null && r.Measurements.Count() == 0, "GetInfluxDBStructureAsync retunred non null or non empty collection"); } catch (Exception e) { Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}"); return; } }
public async Task TestGetInfluxDBStructureAsync_InvalidDB() { try { var client = new InfluxDBClient(influxUrl, dbUName, dbpwd); var r = await client.GetInfluxDBStructureAsync(dbName); Assert.IsTrue(r != null && r.Count >= 0, "GetInfluxDBNamesAsync retunred null or non empty collection"); } catch (Exception e) { Assert.Fail("Unexpected exception of type {0} caught: {1}", e.GetType(), e.Message); return; } }
public async Task TestGetInfluxDBStructureAsync() { try { var client = new InfluxDBClient(influxUrl, dbUName, dbpwd); Stopwatch s = new Stopwatch(); s.Start(); var r = await client.GetInfluxDBStructureAsync(dbName); s.Stop(); Debug.WriteLine(s.ElapsedMilliseconds); Assert.IsTrue(r != null && r.Measurements.Count >= 0, "GetInfluxDBStructureAsync retunred null or non empty collection"); } catch (Exception e) { Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}"); return; } }
public async Task <ProcessStatus> ProcessGenericFile(string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus(); int failedReqCount = 0; try { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); var r = GetFileLayout(InputFileName); if (r.ExitCode != ExitCode.Success) { return(r); } IInfluxDatabase dbStructure; if (settings.GenericFile.Filter != Filters.None) { var filterColumns = new List <GenericColumn>(); if (settings.GenericFile.Filter == Filters.Columns) { if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0) { Logger.LogLine(LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them"); } else { filterColumns = ParseGenericColumns(settings.GenericFile.ColumnsFilter.ToString()); } } dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName); ColumnHeaders = FilterGenericColumns(ColumnHeaders, filterColumns, dbStructure as InfluxDatabase); } var validity = ValidateData(InputFileName); var failureReasons = new Dictionary <Type, FailureTracker>(); List <IInfluxDatapoint> points = new List <IInfluxDatapoint>(), retryQueue = new List <IInfluxDatapoint>(); IInfluxRetentionPolicy policy = null; if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy() { Name = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync(policy)) { throw new InvalidOperationException("Unable to create retention policy"); } } } else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) { throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } } foreach (var line in File.ReadLines(InputFileName).Skip(settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows)) { if (String.IsNullOrWhiteSpace(line) || (!String.IsNullOrEmpty(settings.GenericFile.CommentMarker) && line.StartsWith(settings.GenericFile.CommentMarker))) { continue; } try { result.PointsFound++; var point = ProcessGenericLine(line, ColumnHeaders); if (point == null) { result.PointsFailed++; } else { point.Retention = policy; points.Add(point); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count(p => p.Saved); } else { //add failed to retry queue retryQueue.AddRange(points.Where(p => p.Saved != true)); result.PointsProcessed += points.Count(p => p.Saved); //avoid failing on too many points if (++failedReqCount > 3) { break; } } //a point will be either posted to Influx or in retry queue points.Clear(); } } catch (Exception e) { result.PointsFailed++; var type = e.GetType(); if (e is InfluxDBException) { if ((e as InfluxDBException).Reason == "Partial Write") { retryQueue.AddRange(points.Where(p => p.Saved != true)); result.PointsProcessed += points.Count(p => p.Saved); points.Clear(); } } if (!failureReasons.ContainsKey(type)) { failureReasons.Add(type, new FailureTracker() { ExceptionType = type, Message = e.Message }); } failureReasons[type].LineNumbers.Add(result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1); //avoid too many failures, may be config is wrong if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3) { Logger.LogLine(LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!"); Logger.LogLine(LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!"); break; } } if (result.PointsFailed > 0 || retryQueue.Count > 0) { Logger.Log(LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count); } else { Logger.Log(LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound); } } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) { throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear(); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange(points.Where(p => p.Saved != true)); result.PointsProcessed += points.Count(p => p.Saved); } } //retry all previously failed points if (retryQueue.Count > 0) { Logger.LogLine(LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count); try { if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear(); } else { result.PointsFailed += retryQueue.Count; if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) { throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } } } catch (InfluxDBException e) { if (e.Reason == "Partial Write") { } } } stopwatch.Stop(); if (result.PointsFailed > 0) { Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed); foreach (var f in failureReasons.Values) { Logger.LogLine(LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers.Take(5)), f.ExceptionType, f.Message); } if (result.PointsFailed == result.PointsFound) { result.ExitCode = ExitCode.UnableToProcess; } else { result.ExitCode = ExitCode.ProcessedWithErrors; } } else { result.ExitCode = ExitCode.Success; Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound); } } catch (Exception e) { Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnableToProcess; } return(result); }
public async Task <ProcessStatus> ProcessPerfMonLog(string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus(); int linesProcessed = 0; int failedLines = 0; int failedReqCount = 0; try { var failureReasons = new Dictionary <Type, FailureTracker>(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); var firstLine = File.ReadLines(InputFileName).FirstOrDefault(); var firstCol = firstLine.Substring(0, firstLine.IndexOf(',')); if (!firstCol.Contains("PDH-CSV")) { throw new InvalidDataException("Input file is not a Standard Perfmon csv file"); } var x = Regex.Matches(firstCol, "([-0-9]+)"); if (x.Count > 0) { minOffset = int.Parse(x[3].ToString()); } //get the column headers List <PerfmonCounter> pecrfCounters; try { pecrfCounters = ParsePerfMonFileHeader(firstLine); } catch (Exception ex) { throw new InvalidDataException("Unable to parse file headers", ex); } IInfluxDatabase dbStructure; IEnumerable <IGrouping <string, PerfmonCounter> > perfGroup; if (settings.PerfmonFile.Filter != Filters.None) { var filterColumns = ParsePerfMonFileHeader(settings.PerfmonFile.ColumnsFilter.ToString(), false); dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName); perfGroup = FilterPerfmonLogColumns(pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy(p => p.PerformanceObject); } else { perfGroup = pecrfCounters.GroupBy(p => p.PerformanceObject); } List <IInfluxDatapoint> points = null, retryQueue = new List <IInfluxDatapoint>(); if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy() { Name = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync(policy)) { throw new InvalidOperationException("Unable to create retention policy"); } } } else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) { throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } } //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) => foreach (var line in File.ReadLines(InputFileName).Skip(1)) { try { var linePoints = ProcessPerfmonLogLine(line, perfGroup); linesProcessed++; if (linePoints == null || linePoints.Count == 0) { failedLines++; } else { result.PointsFound += linePoints.Count; if (points == null) { points = linePoints; } else { points.AddRange(linePoints); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count(p => p.Saved); } else { //add failed to retry queue retryQueue.AddRange(points.Where(p => p.Saved != true)); result.PointsProcessed += points.Count(p => p.Saved); //avoid failing on too many points if (++failedReqCount > 4) { break; } } //a point will be either posted to Influx or in retry queue points = null; } } } catch (Exception e) { failedLines++; var type = e.GetType(); if (!failureReasons.ContainsKey(type)) { failureReasons.Add(type, new FailureTracker() { ExceptionType = type, Message = e.Message }); } failureReasons[type].LineNumbers.Add(linesProcessed); } if (failedLines > 0 || retryQueue.Count > 0) { Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); } else { Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound); } } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) { throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear(); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange(points.Where(p => p.Saved != true)); result.PointsProcessed += points.Count(p => p.Saved); } } if (retryQueue.Count > 0) { Logger.LogLine(LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear(); } else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) { throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } else { result.PointsFailed += retryQueue.Count; } } pecrfCounters.Clear(); stopwatch.Stop(); if (failedLines > 0 || retryQueue.Count > 0) { Logger.Log(LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count); foreach (var f in failureReasons.Values) { Logger.LogLine(LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers), f.ExceptionType, f.Message); } if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count) { result.ExitCode = ExitCode.UnableToProcess; } else { result.ExitCode = ExitCode.ProcessedWithErrors; } } else { result.ExitCode = ExitCode.Success; Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound); } } catch (Exception e) { Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnknownError; } return(result); }