public GetInfluxDBStructureAsync ( string dbName ) : Task |
||
dbName | string | Name of the database |
return | Task |
public async Task TestGetInfluxDBStructureAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.GetInfluxDBStructureAsync ("InvalidDB"); Assert.IsTrue (r != null && r.Count == 0, "GetInfluxDBNamesAsync retunred null or non empty collection"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int failedReqCount = 0; try { Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var r = GetFileLayout (InputFileName); if (r.ExitCode != ExitCode.Success) return r; IInfluxDatabase dbStructure; if (settings.GenericFile.Filter != Filters.None) { var filterColumns = new List<GenericColumn> (); if (settings.GenericFile.Filter == Filters.Columns) { if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0) Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them"); else filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ()); } dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase); } var validity = ValidateData (InputFileName); var failureReasons = new Dictionary<Type, FailureTracker> (); List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> (); IInfluxRetentionPolicy policy = null; if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows)) { if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker))) continue; try { result.PointsFound++; var point = ProcessGenericLine (line, ColumnHeaders); if (point == null) result.PointsFailed++; else { point.Retention = policy; points.Add (point); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 3) break; } //a point will be either posted to Influx or in retry queue points.Clear (); } } catch (Exception e) { result.PointsFailed++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1); //avoid too many failures, may be config is wrong if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3) { Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!"); Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!"); break; } } if (result.PointsFailed > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } //retry all previously failed points if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else { result.PointsFailed += retryQueue.Count; if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } } stopwatch.Stop (); if (result.PointsFailed > 0) { Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message); if (result.PointsFailed == result.PointsFound) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnableToProcess; } return result; }
public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int linesProcessed = 0; int failedLines = 0; int failedReqCount = 0; try { var failureReasons = new Dictionary<Type, FailureTracker> (); Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var firstLine = File.ReadLines (InputFileName).FirstOrDefault (); var firstCol = firstLine.Substring (0, firstLine.IndexOf (',')); if (!firstCol.Contains ("PDH-CSV")) throw new InvalidDataException ("Input file is not a Standard Perfmon csv file"); var x = Regex.Matches (firstCol, "([-0-9]+)"); if (x.Count > 0) minOffset = int.Parse (x[3].ToString ()); //get the column headers List<PerfmonCounter> pecrfCounters; try { pecrfCounters = ParsePerfMonFileHeader (firstLine); } catch (Exception ex) { throw new InvalidDataException ("Unable to parse file headers", ex); } IInfluxDatabase dbStructure; IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup; if (settings.PerfmonFile.Filter != Filters.None) { var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false); dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject); } else { perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject); } List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> (); if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) => foreach (var line in File.ReadLines (InputFileName).Skip (1)) { try { var linePoints = ProcessPerfmonLogLine (line, perfGroup); linesProcessed++; if (linePoints == null || linePoints.Count == 0) failedLines++; else { result.PointsFound += linePoints.Count; if (points == null) points = linePoints; else points.AddRange (linePoints); if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 4) break; } //a point will be either posted to Influx or in retry queue points = null; } } } catch (Exception e) { failedLines++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (linesProcessed); } if (failedLines > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); else { result.PointsFailed += retryQueue.Count; } } pecrfCounters.Clear (); stopwatch.Stop (); if (failedLines > 0 || retryQueue.Count > 0) { Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message); if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnknownError; } return result; }