public PostPointsAsync ( string dbName, IEnumerable |
||
dbName | string | InfluxDB database name |
Points | IEnumerable |
Collection of Influx data points to be written |
Résultat | Task |
public async Task TestBachingPostPointsAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var points = new List<IInfluxDatapoint> (); var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var start = DateTime.Now.AddDays (-5); var end = DateTime.Now; for ( int i = 0; i < 5000; i++ ) { var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DataGen.RandomDate (start, end); valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DataGen.RandomDate (start, end); valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valBool = new InfluxDatapoint<bool> (); valBool.UtcTimestamp = DataGen.RandomDate (start, end); valBool.Tags.Add ("TestDate", today); valBool.Tags.Add ("TestTime", now); valBool.Fields.Add ("Booleanfield", DateTime.Now.Ticks % 2 == 0); valBool.MeasurementName = measurementName; valBool.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valString = new InfluxDatapoint<string> (); valString.UtcTimestamp = DataGen.RandomDate (start, end); valString.Tags.Add ("TestDate", today); valString.Tags.Add ("TestTime", now); valString.Fields.Add ("Stringfield", DataGen.RandomString ()); valString.MeasurementName = measurementName; valString.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; points.Add (valString); points.Add (valInt); points.Add (valDouble); points.Add (valBool); } var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (points.TrueForAll (p => p.Saved == true), "PostPointsAsync did not save all points"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestPostPointsAsync_PartialWrite() { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var points = new List<IInfluxDatapoint> (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", Double.NaN); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; points.Add (valDouble); for ( int i = 0; i < 5; i++ ) { var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow.AddSeconds (-1 * DataGen.RandomInt (3600)); valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Seconds; points.Add (valInt); } valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", Double.NaN); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; points.Add (valDouble); var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (r, "PostPointsAsync retunred false"); }
public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int failedReqCount = 0; try { Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var r = GetFileLayout (InputFileName); if (r.ExitCode != ExitCode.Success) return r; IInfluxDatabase dbStructure; if (settings.GenericFile.Filter != Filters.None) { var filterColumns = new List<GenericColumn> (); if (settings.GenericFile.Filter == Filters.Columns) { if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0) Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them"); else filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ()); } dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase); } var validity = ValidateData (InputFileName); var failureReasons = new Dictionary<Type, FailureTracker> (); List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> (); IInfluxRetentionPolicy policy = null; if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows)) { if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker))) continue; try { result.PointsFound++; var point = ProcessGenericLine (line, ColumnHeaders); if (point == null) result.PointsFailed++; else { point.Retention = policy; points.Add (point); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 3) break; } //a point will be either posted to Influx or in retry queue points.Clear (); } } catch (Exception e) { result.PointsFailed++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1); //avoid too many failures, may be config is wrong if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3) { Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!"); Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!"); break; } } if (result.PointsFailed > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } //retry all previously failed points if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else { result.PointsFailed += retryQueue.Count; if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } } stopwatch.Stop (); if (result.PointsFailed > 0) { Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message); if (result.PointsFailed == result.PointsFound) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnableToProcess; } return result; }
public async Task TestPostPointsAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var points = new List<IInfluxDatapoint> (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Nanoseconds; points.Add (valDouble); valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Microseconds; points.Add (valDouble); var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow; valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Milliseconds; points.Add (valInt); valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow; valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Seconds; points.Add (valInt); var valBool = new InfluxDatapoint<bool> (); valBool.UtcTimestamp = DateTime.UtcNow; valBool.Tags.Add ("TestDate", today); valBool.Tags.Add ("TestTime", now); valBool.Fields.Add ("Booleanfield", time.Ticks % 2 == 0); valBool.MeasurementName = measurementName; valBool.Precision = TimePrecision.Minutes; points.Add (valBool); var valString = new InfluxDatapoint<string> (); valString.UtcTimestamp = DateTime.UtcNow; valString.Tags.Add ("TestDate", today); valString.Tags.Add ("TestTime", now); valString.Fields.Add ("Stringfield", DataGen.RandomString ()); valString.MeasurementName = measurementName; valString.Precision = TimePrecision.Hours; points.Add (valString); var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (r, "PostPointsAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int linesProcessed = 0; int failedLines = 0; int failedReqCount = 0; try { var failureReasons = new Dictionary<Type, FailureTracker> (); Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var firstLine = File.ReadLines (InputFileName).FirstOrDefault (); var firstCol = firstLine.Substring (0, firstLine.IndexOf (',')); if (!firstCol.Contains ("PDH-CSV")) throw new InvalidDataException ("Input file is not a Standard Perfmon csv file"); var x = Regex.Matches (firstCol, "([-0-9]+)"); if (x.Count > 0) minOffset = int.Parse (x[3].ToString ()); //get the column headers List<PerfmonCounter> pecrfCounters; try { pecrfCounters = ParsePerfMonFileHeader (firstLine); } catch (Exception ex) { throw new InvalidDataException ("Unable to parse file headers", ex); } IInfluxDatabase dbStructure; IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup; if (settings.PerfmonFile.Filter != Filters.None) { var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false); dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject); } else { perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject); } List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> (); if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) => foreach (var line in File.ReadLines (InputFileName).Skip (1)) { try { var linePoints = ProcessPerfmonLogLine (line, perfGroup); linesProcessed++; if (linePoints == null || linePoints.Count == 0) failedLines++; else { result.PointsFound += linePoints.Count; if (points == null) points = linePoints; else points.AddRange (linePoints); if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 4) break; } //a point will be either posted to Influx or in retry queue points = null; } } } catch (Exception e) { failedLines++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (linesProcessed); } if (failedLines > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); else { result.PointsFailed += retryQueue.Count; } } pecrfCounters.Clear (); stopwatch.Stop (); if (failedLines > 0 || retryQueue.Count > 0) { Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message); if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnknownError; } return result; }