private static async Task<InfluxDBClient> GetClientAsync (InfluxerConfigSection settings) { var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password); var dbNames = await client.GetInfluxDBNamesAsync (); if (dbNames.Contains (settings.InfluxDB.DatabaseName)) return client; else { await client.CreateDatabaseAsync (settings.InfluxDB.DatabaseName); return client; } }
public async Task TestGetInfluxDBStructureAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.GetInfluxDBStructureAsync ("InvalidDB"); Assert.IsTrue (r != null && r.Count == 0, "GetInfluxDBNamesAsync retunred null or non empty collection"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task <double> SumTheWaterLevel(string endpoint, string database) { var client = new InfluxDBClient(endpoint); var query = "SELECT water_level FROM h2o_feet"; var results = await client.QueryMultiSeriesAsync(database, query); var serie = results.Single(); double total = 0; foreach (dynamic entry in serie.Entries) { // Not sure why but water_level comes back as a string. total += double.Parse(entry.Water_level); } return(total); }
public async Task TestPostBooleanPointAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var valBool = new InfluxDatapoint<bool> (); valBool.UtcTimestamp = DateTime.UtcNow; valBool.Tags.Add ("TestDate", time.ToShortDateString ()); valBool.Tags.Add ("TestTime", time.ToShortTimeString ()); valBool.Fields.Add ("Booleanfield", time.Ticks % 2 == 0); valBool.MeasurementName = measurementName; valBool.Precision = TimePrecision.Seconds; var r = await client.PostPointAsync (dbName, valBool); Assert.IsTrue (r, "PostPointAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestPostStringPointAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var valString = new InfluxDatapoint<string> (); valString.UtcTimestamp = DateTime.UtcNow; valString.Tags.Add ("TestDate", time.ToShortDateString ()); valString.Tags.Add ("TestTime", time.ToShortTimeString ()); valString.Fields.Add ("Stringfield", @"j0,Oox7ju6lJvA0\");//DataGen.RandomString ()); valString.MeasurementName = measurementName; valString.Precision = TimePrecision.Seconds; var r = await client.PostPointAsync (dbName, valString); Assert.IsTrue (r, "PostPointAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestPostDoublePointAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var rand = new Random (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", time.ToShortDateString ()); valDouble.Tags.Add ("TestTime", time.ToShortTimeString ()); valDouble.Fields.Add ("Doublefield", rand.NextDouble ()); valDouble.Fields.Add ("Doublefield2", rand.NextDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; var r = await client.PostPointAsync (dbName, valDouble); Assert.IsTrue (r, "PostPointAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
private static async Task<bool> VerifyDatabaseAsync (InfluxDBClient client, string DBName) { try { //verify DB exists, create if not var dbNames = await client.GetInfluxDBNamesAsync (); if (dbNames.Contains (DBName)) return true; else { var filter = settings.FileFormat == FileFormats.Perfmon ? settings.PerfmonFile.Filter : settings.GenericFile.Filter; if (filter == Filters.Measurement || filter == Filters.Field) { Logger.LogLine (LogLevel.Info, "Measurement/Field filtering is not applicable for new database!!"); filter = Filters.None; } return await client.CreateDatabaseAsync (DBName); } } catch (Exception e) { Logger.LogLine (LogLevel.Info, "Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); } return false; }
public async Task TestBachingPostPointsAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var points = new List<IInfluxDatapoint> (); var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var start = DateTime.Now.AddDays (-5); var end = DateTime.Now; for ( int i = 0; i < 5000; i++ ) { var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DataGen.RandomDate (start, end); valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DataGen.RandomDate (start, end); valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valBool = new InfluxDatapoint<bool> (); valBool.UtcTimestamp = DataGen.RandomDate (start, end); valBool.Tags.Add ("TestDate", today); valBool.Tags.Add ("TestTime", now); valBool.Fields.Add ("Booleanfield", DateTime.Now.Ticks % 2 == 0); valBool.MeasurementName = measurementName; valBool.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; var valString = new InfluxDatapoint<string> (); valString.UtcTimestamp = DataGen.RandomDate (start, end); valString.Tags.Add ("TestDate", today); valString.Tags.Add ("TestTime", now); valString.Fields.Add ("Stringfield", DataGen.RandomString ()); valString.MeasurementName = measurementName; valString.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1; points.Add (valString); points.Add (valInt); points.Add (valDouble); points.Add (valBool); } var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (points.TrueForAll (p => p.Saved == true), "PostPointsAsync did not save all points"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestPostPointsAsync_PartialWrite() { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var points = new List<IInfluxDatapoint> (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", Double.NaN); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; points.Add (valDouble); for ( int i = 0; i < 5; i++ ) { var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow.AddSeconds (-1 * DataGen.RandomInt (3600)); valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Seconds; points.Add (valInt); } valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", Double.NaN); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; points.Add (valDouble); var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (r, "PostPointsAsync retunred false"); }
public async Task TestGetInfluxDBNamesAsync_Auth() { var client = new InfluxDBClient (influxUrl); var r = await client.GetInfluxDBNamesAsync (); }
public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int failedReqCount = 0; try { Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var r = GetFileLayout (InputFileName); if (r.ExitCode != ExitCode.Success) return r; IInfluxDatabase dbStructure; if (settings.GenericFile.Filter != Filters.None) { var filterColumns = new List<GenericColumn> (); if (settings.GenericFile.Filter == Filters.Columns) { if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0) Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them"); else filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ()); } dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase); } var validity = ValidateData (InputFileName); var failureReasons = new Dictionary<Type, FailureTracker> (); List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> (); IInfluxRetentionPolicy policy = null; if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows)) { if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker))) continue; try { result.PointsFound++; var point = ProcessGenericLine (line, ColumnHeaders); if (point == null) result.PointsFailed++; else { point.Retention = policy; points.Add (point); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 3) break; } //a point will be either posted to Influx or in retry queue points.Clear (); } } catch (Exception e) { result.PointsFailed++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1); //avoid too many failures, may be config is wrong if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3) { Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!"); Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!"); break; } } if (result.PointsFailed > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } //retry all previously failed points if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else { result.PointsFailed += retryQueue.Count; if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } } stopwatch.Stop (); if (result.PointsFailed > 0) { Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message); if (result.PointsFailed == result.PointsFound) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnableToProcess; } return result; }
public async Task TestPostValuesAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var val = new Random (); var values = new Dictionary<string, double> () { { "filed1", val.NextDouble () * 10 }, { "filed2", val.NextDouble () * 10 }, { "filed3", val.NextDouble () * 10 } }; var r = await client.PostValuesAsync (dbName, measurementName, DateTime.UtcNow.ToEpoch (TimePrecision.Seconds), TimePrecision.Seconds, "testTag=testTagValue",values); Assert.IsTrue (r, "PostValuesAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestPostValueAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.PostValueAsync (dbName, measurementName, DateTime.UtcNow.ToEpoch(TimePrecision.Seconds),TimePrecision.Seconds,"testTag=testTagValue","Temp",33.05); Assert.IsTrue (r, "PostValueAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
static int Main (string[] args) { try { if (!CommandLineProcessor.ProcessArguments (args)) return (int) ExitCode.Success; settings = CommandLineProcessor.Settings; } catch (ArgumentException e) { Logger.LogLine (LogLevel.Error, e.Message); return (int) ExitCode.InvalidArgument; } catch (FileLoadException e) { Logger.LogLine (LogLevel.Error, e.Message); Logger.LogLine (LogLevel.Info, "Problem loading config file, regenerate it with /config option"); return (int) ExitCode.InvalidFilename; } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Error processing arguments {0}: {1}", e.GetType ().Name, e.Message); return (int) ExitCode.InvalidArgument; } #region Validate inputs if (String.IsNullOrWhiteSpace (settings.InputFileName)) { Logger.LogLine (LogLevel.Error, "Input File Name is not specified!! Can't continue"); return (int) ExitCode.InvalidArgument; } try { settings.InputFileName = Path.GetFullPath (settings.InputFileName); } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Error with input file:{0},{1}", e.GetType ().Name, e.Message); Logger.LogLine (LogLevel.Info, "Problem with inputfile name, check path"); return (int) ExitCode.InvalidFilename; } if (String.IsNullOrWhiteSpace (settings.InfluxDB.InfluxUri)) { Logger.LogLine (LogLevel.Error, "Influx DB Uri is not configured!!"); return (int) ExitCode.InvalidArgument; } if (String.IsNullOrWhiteSpace (settings.InfluxDB.DatabaseName)) { Logger.LogLine (LogLevel.Error, "Influx DB name is not configured!!"); return (int) ExitCode.InvalidArgument; } #endregion ProcessStatus result = new ProcessStatus () { ExitCode = ExitCode.UnknownError }; try { Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password); if (!VerifyDatabaseAsync (client, settings.InfluxDB.DatabaseName).Result) { Logger.LogLine (LogLevel.Info, "Unable to create DB {0}", settings.InfluxDB.DatabaseName); return (int) ExitCode.UnableToProcess; } switch (settings.FileFormat) { case FileFormats.Perfmon: result = new PerfmonFile ().ProcessPerfMonLog (settings.InputFileName, client).Result; break; case FileFormats.Generic: if (String.IsNullOrWhiteSpace (settings.InfluxDB.Measurement)) throw new ArgumentException ("Generic format needs TableName input"); result = new GenericFile ().ProcessGenericFile (settings.InputFileName, client).Result; break; } stopwatch.Stop (); Logger.LogLine (LogLevel.Info, "\n Finished!! Processed {0} points (Success: {1}, Failed:{2}) in {3}", result.PointsFound, result.PointsProcessed, result.PointsFailed, stopwatch.Elapsed.ToString ()); } catch (AggregateException e) { Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.InnerException.GetType ().Name, e.InnerException.Message, e.InnerException.StackTrace); } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); } return (int) result.ExitCode; }
public async Task TestGetInfluxDBNamesAsync_ServiceUnavailable() { var client = new InfluxDBClient (invalidInfluxUrl); var r = await client.GetInfluxDBNamesAsync (); }
public async Task TestPostPointsAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var points = new List<IInfluxDatapoint> (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Nanoseconds; points.Add (valDouble); valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ()); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Microseconds; points.Add (valDouble); var valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow; valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Milliseconds; points.Add (valInt); valInt = new InfluxDatapoint<int> (); valInt.UtcTimestamp = DateTime.UtcNow; valInt.Tags.Add ("TestDate", today); valInt.Tags.Add ("TestTime", now); valInt.Fields.Add ("Intfield", DataGen.RandomInt ()); valInt.Fields.Add ("Intfield2", DataGen.RandomInt ()); valInt.MeasurementName = measurementName; valInt.Precision = TimePrecision.Seconds; points.Add (valInt); var valBool = new InfluxDatapoint<bool> (); valBool.UtcTimestamp = DateTime.UtcNow; valBool.Tags.Add ("TestDate", today); valBool.Tags.Add ("TestTime", now); valBool.Fields.Add ("Booleanfield", time.Ticks % 2 == 0); valBool.MeasurementName = measurementName; valBool.Precision = TimePrecision.Minutes; points.Add (valBool); var valString = new InfluxDatapoint<string> (); valString.UtcTimestamp = DateTime.UtcNow; valString.Tags.Add ("TestDate", today); valString.Tags.Add ("TestTime", now); valString.Fields.Add ("Stringfield", DataGen.RandomString ()); valString.MeasurementName = measurementName; valString.Precision = TimePrecision.Hours; points.Add (valString); var r = await client.PostPointsAsync (dbName, points); Assert.IsTrue (r, "PostPointsAsync retunred false"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestCreateDatabaseAsync_InvalidName() { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.CreateDatabaseAsync (invalidDbName); }
public async Task TestPostPointAsync_InvalidReq() { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var time = DateTime.Now; var today = DateTime.Now.ToShortDateString (); var now = DateTime.Now.ToShortTimeString (); var valDouble = new InfluxDatapoint<double> (); valDouble.UtcTimestamp = DateTime.UtcNow; valDouble.Tags.Add ("TestDate", today); valDouble.Tags.Add ("TestTime", now); valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ()); valDouble.Fields.Add ("Doublefield2", Double.NaN); valDouble.MeasurementName = measurementName; valDouble.Precision = TimePrecision.Seconds; var r = await client.PostPointAsync (dbName, valDouble); Assert.IsTrue (r, "PostPointsAsync retunred false"); }
public async Task TestCreateDatabaseAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.CreateDatabaseAsync (dbName); Assert.IsTrue (r, "CreateDatabaseAsync retunred false"); } catch ( InvalidOperationException e ) { StringAssert.Equals (e.Message, "database already exists"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestGetInfluxDBNamesAsync_Auth2() { var client = new InfluxDBClient (influxUrl, invalidUName, dbpwd); var r = await client.GetInfluxDBNamesAsync (); }
public async Task TestQueryAsync() { try { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.QueryDBAsync ("stress", "select * from performance limit 10"); DateTime d; Assert.IsTrue (r != null && DateTime.TryParse (r[0].time, out d), "QueryDBAsync retunred null or invalid data"); } catch ( Exception e ) { Assert.Fail ("Unexpected exception of type {0} caught: {1}", e.GetType (), e.Message); return; } }
public async Task TestQueryAsync_MultiSeries() { var client = new InfluxDBClient (influxUrl, dbUName, dbpwd); var r = await client.QueryDBAsync ("_internal", "Show Series"); }
public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int linesProcessed = 0; int failedLines = 0; int failedReqCount = 0; try { var failureReasons = new Dictionary<Type, FailureTracker> (); Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var firstLine = File.ReadLines (InputFileName).FirstOrDefault (); var firstCol = firstLine.Substring (0, firstLine.IndexOf (',')); if (!firstCol.Contains ("PDH-CSV")) throw new InvalidDataException ("Input file is not a Standard Perfmon csv file"); var x = Regex.Matches (firstCol, "([-0-9]+)"); if (x.Count > 0) minOffset = int.Parse (x[3].ToString ()); //get the column headers List<PerfmonCounter> pecrfCounters; try { pecrfCounters = ParsePerfMonFileHeader (firstLine); } catch (Exception ex) { throw new InvalidDataException ("Unable to parse file headers", ex); } IInfluxDatabase dbStructure; IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup; if (settings.PerfmonFile.Filter != Filters.None) { var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false); dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject); } else { perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject); } List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> (); if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) => foreach (var line in File.ReadLines (InputFileName).Skip (1)) { try { var linePoints = ProcessPerfmonLogLine (line, perfGroup); linesProcessed++; if (linePoints == null || linePoints.Count == 0) failedLines++; else { result.PointsFound += linePoints.Count; if (points == null) points = linePoints; else points.AddRange (linePoints); if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 4) break; } //a point will be either posted to Influx or in retry queue points = null; } } } catch (Exception e) { failedLines++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (linesProcessed); } if (failedLines > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); else { result.PointsFailed += retryQueue.Count; } } pecrfCounters.Clear (); stopwatch.Stop (); if (failedLines > 0 || retryQueue.Count > 0) { Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count); Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message); if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnknownError; } return result; }