private static async Task<InfluxDBClient> GetClientAsync (InfluxerConfigSection settings)
{
var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);
var dbNames = await client.GetInfluxDBNamesAsync ();
if (dbNames.Contains (settings.InfluxDB.DatabaseName))
return client;
else
{
await client.CreateDatabaseAsync (settings.InfluxDB.DatabaseName);
return client;
}
}
public async Task TestGetInfluxDBStructureAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.GetInfluxDBStructureAsync ("InvalidDB");
Assert.IsTrue (r != null && r.Count == 0, "GetInfluxDBNamesAsync retunred null or non empty collection");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task <double> SumTheWaterLevel(string endpoint, string database)
{
var client = new InfluxDBClient(endpoint);
var query = "SELECT water_level FROM h2o_feet";
var results = await client.QueryMultiSeriesAsync(database, query);
var serie = results.Single();
double total = 0;
foreach (dynamic entry in serie.Entries)
{
// Not sure why but water_level comes back as a string.
total += double.Parse(entry.Water_level);
}
return(total);
}
public async Task TestPostBooleanPointAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var valBool = new InfluxDatapoint<bool> ();
valBool.UtcTimestamp = DateTime.UtcNow;
valBool.Tags.Add ("TestDate", time.ToShortDateString ());
valBool.Tags.Add ("TestTime", time.ToShortTimeString ());
valBool.Fields.Add ("Booleanfield", time.Ticks % 2 == 0);
valBool.MeasurementName = measurementName;
valBool.Precision = TimePrecision.Seconds;
var r = await client.PostPointAsync (dbName, valBool);
Assert.IsTrue (r, "PostPointAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestPostStringPointAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var valString = new InfluxDatapoint<string> ();
valString.UtcTimestamp = DateTime.UtcNow;
valString.Tags.Add ("TestDate", time.ToShortDateString ());
valString.Tags.Add ("TestTime", time.ToShortTimeString ());
valString.Fields.Add ("Stringfield", @"j0,Oox7ju6lJvA0\");//DataGen.RandomString ());
valString.MeasurementName = measurementName;
valString.Precision = TimePrecision.Seconds;
var r = await client.PostPointAsync (dbName, valString);
Assert.IsTrue (r, "PostPointAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestPostDoublePointAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var rand = new Random ();
var valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", time.ToShortDateString ());
valDouble.Tags.Add ("TestTime", time.ToShortTimeString ());
valDouble.Fields.Add ("Doublefield", rand.NextDouble ());
valDouble.Fields.Add ("Doublefield2", rand.NextDouble ());
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Seconds;
var r = await client.PostPointAsync (dbName, valDouble);
Assert.IsTrue (r, "PostPointAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
private static async Task<bool> VerifyDatabaseAsync (InfluxDBClient client, string DBName)
{
try
{
//verify DB exists, create if not
var dbNames = await client.GetInfluxDBNamesAsync ();
if (dbNames.Contains (DBName))
return true;
else
{
var filter = settings.FileFormat == FileFormats.Perfmon ? settings.PerfmonFile.Filter : settings.GenericFile.Filter;
if (filter == Filters.Measurement || filter == Filters.Field)
{
Logger.LogLine (LogLevel.Info, "Measurement/Field filtering is not applicable for new database!!");
filter = Filters.None;
}
return await client.CreateDatabaseAsync (DBName);
}
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Info, "Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
}
return false;
}
public async Task TestBachingPostPointsAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var points = new List<IInfluxDatapoint> ();
var today = DateTime.Now.ToShortDateString ();
var now = DateTime.Now.ToShortTimeString ();
var start = DateTime.Now.AddDays (-5);
var end = DateTime.Now;
for ( int i = 0; i < 5000; i++ )
{
var valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DataGen.RandomDate (start, end);
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ());
valDouble.MeasurementName = measurementName;
valDouble.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1;
var valInt = new InfluxDatapoint<int> ();
valInt.UtcTimestamp = DataGen.RandomDate (start, end);
valInt.Tags.Add ("TestDate", today);
valInt.Tags.Add ("TestTime", now);
valInt.Fields.Add ("Intfield", DataGen.RandomInt ());
valInt.Fields.Add ("Intfield2", DataGen.RandomInt ());
valInt.MeasurementName = measurementName;
valInt.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1;
var valBool = new InfluxDatapoint<bool> ();
valBool.UtcTimestamp = DataGen.RandomDate (start, end);
valBool.Tags.Add ("TestDate", today);
valBool.Tags.Add ("TestTime", now);
valBool.Fields.Add ("Booleanfield", DateTime.Now.Ticks % 2 == 0);
valBool.MeasurementName = measurementName;
valBool.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1;
var valString = new InfluxDatapoint<string> ();
valString.UtcTimestamp = DataGen.RandomDate (start, end);
valString.Tags.Add ("TestDate", today);
valString.Tags.Add ("TestTime", now);
valString.Fields.Add ("Stringfield", DataGen.RandomString ());
valString.MeasurementName = measurementName;
valString.Precision = (TimePrecision) ( DataGen.RandomInt () % 6 ) + 1;
points.Add (valString);
points.Add (valInt);
points.Add (valDouble);
points.Add (valBool);
}
var r = await client.PostPointsAsync (dbName, points);
Assert.IsTrue (points.TrueForAll (p => p.Saved == true), "PostPointsAsync did not save all points");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestPostPointsAsync_PartialWrite()
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var today = DateTime.Now.ToShortDateString ();
var now = DateTime.Now.ToShortTimeString ();
var points = new List<IInfluxDatapoint> ();
var valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", Double.NaN);
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Seconds;
points.Add (valDouble);
for ( int i = 0; i < 5; i++ )
{
var valInt = new InfluxDatapoint<int> ();
valInt.UtcTimestamp = DateTime.UtcNow.AddSeconds (-1 * DataGen.RandomInt (3600));
valInt.Tags.Add ("TestDate", today);
valInt.Tags.Add ("TestTime", now);
valInt.Fields.Add ("Intfield", DataGen.RandomInt ());
valInt.Fields.Add ("Intfield2", DataGen.RandomInt ());
valInt.MeasurementName = measurementName;
valInt.Precision = TimePrecision.Seconds;
points.Add (valInt);
}
valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", Double.NaN);
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Seconds;
points.Add (valDouble);
var r = await client.PostPointsAsync (dbName, points);
Assert.IsTrue (r, "PostPointsAsync retunred false");
}
public async Task TestGetInfluxDBNamesAsync_Auth()
{
var client = new InfluxDBClient (influxUrl);
var r = await client.GetInfluxDBNamesAsync ();
}
public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus ();
int failedReqCount = 0;
try
{
Stopwatch stopwatch = new Stopwatch ();
stopwatch.Start ();
var r = GetFileLayout (InputFileName);
if (r.ExitCode != ExitCode.Success)
return r;
IInfluxDatabase dbStructure;
if (settings.GenericFile.Filter != Filters.None)
{
var filterColumns = new List<GenericColumn> ();
if (settings.GenericFile.Filter == Filters.Columns)
{
if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
else
filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ());
}
dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);
}
var validity = ValidateData (InputFileName);
var failureReasons = new Dictionary<Type, FailureTracker> ();
List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> ();
IInfluxRetentionPolicy policy = null;
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy ()
{
Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync (policy))
throw new InvalidOperationException ("Unable to create retention policy");
}
}
else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
{
if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker)))
continue;
try
{
result.PointsFound++;
var point = ProcessGenericLine (line, ColumnHeaders);
if (point == null)
result.PointsFailed++;
else
{
point.Retention = policy;
points.Add (point);
}
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count;
}
else
{
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 3)
break;
}
//a point will be either posted to Influx or in retry queue
points.Clear ();
}
}
catch (Exception e)
{
result.PointsFailed++;
var type = e.GetType ();
if (!failureReasons.ContainsKey (type))
failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);
//avoid too many failures, may be config is wrong
if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
{
Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
break;
}
}
if (result.PointsFailed > 0 || retryQueue.Count > 0)
Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
else
Logger.Log (LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound);
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear ();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
}
}
//retry all previously failed points
if (retryQueue.Count > 0)
{
Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear ();
}
else
{
result.PointsFailed += retryQueue.Count;
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
}
stopwatch.Stop ();
if (result.PointsFailed > 0)
{
Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
foreach (var f in failureReasons.Values)
Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message);
if (result.PointsFailed == result.PointsFound)
result.ExitCode = ExitCode.UnableToProcess;
else
result.ExitCode = ExitCode.ProcessedWithErrors;
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnableToProcess;
}
return result;
}
public async Task TestPostValuesAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var val = new Random ();
var values = new Dictionary<string, double> () { { "filed1", val.NextDouble () * 10 }, { "filed2", val.NextDouble () * 10 }, { "filed3", val.NextDouble () * 10 } };
var r = await client.PostValuesAsync (dbName, measurementName, DateTime.UtcNow.ToEpoch (TimePrecision.Seconds), TimePrecision.Seconds, "testTag=testTagValue",values);
Assert.IsTrue (r, "PostValuesAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestPostValueAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.PostValueAsync (dbName, measurementName, DateTime.UtcNow.ToEpoch(TimePrecision.Seconds),TimePrecision.Seconds,"testTag=testTagValue","Temp",33.05);
Assert.IsTrue (r, "PostValueAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
static int Main (string[] args)
{
try
{
if (!CommandLineProcessor.ProcessArguments (args))
return (int) ExitCode.Success;
settings = CommandLineProcessor.Settings;
}
catch (ArgumentException e)
{
Logger.LogLine (LogLevel.Error, e.Message);
return (int) ExitCode.InvalidArgument;
}
catch (FileLoadException e)
{
Logger.LogLine (LogLevel.Error, e.Message);
Logger.LogLine (LogLevel.Info, "Problem loading config file, regenerate it with /config option");
return (int) ExitCode.InvalidFilename;
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Error processing arguments {0}: {1}", e.GetType ().Name, e.Message);
return (int) ExitCode.InvalidArgument;
}
#region Validate inputs
if (String.IsNullOrWhiteSpace (settings.InputFileName))
{
Logger.LogLine (LogLevel.Error, "Input File Name is not specified!! Can't continue");
return (int) ExitCode.InvalidArgument;
}
try
{
settings.InputFileName = Path.GetFullPath (settings.InputFileName);
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Error with input file:{0},{1}", e.GetType ().Name, e.Message);
Logger.LogLine (LogLevel.Info, "Problem with inputfile name, check path");
return (int) ExitCode.InvalidFilename;
}
if (String.IsNullOrWhiteSpace (settings.InfluxDB.InfluxUri))
{
Logger.LogLine (LogLevel.Error, "Influx DB Uri is not configured!!");
return (int) ExitCode.InvalidArgument;
}
if (String.IsNullOrWhiteSpace (settings.InfluxDB.DatabaseName))
{
Logger.LogLine (LogLevel.Error, "Influx DB name is not configured!!");
return (int) ExitCode.InvalidArgument;
}
#endregion
ProcessStatus result = new ProcessStatus () { ExitCode = ExitCode.UnknownError };
try
{
Stopwatch stopwatch = new Stopwatch ();
stopwatch.Start ();
var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);
if (!VerifyDatabaseAsync (client, settings.InfluxDB.DatabaseName).Result)
{
Logger.LogLine (LogLevel.Info, "Unable to create DB {0}", settings.InfluxDB.DatabaseName);
return (int) ExitCode.UnableToProcess;
}
switch (settings.FileFormat)
{
case FileFormats.Perfmon:
result = new PerfmonFile ().ProcessPerfMonLog (settings.InputFileName, client).Result;
break;
case FileFormats.Generic:
if (String.IsNullOrWhiteSpace (settings.InfluxDB.Measurement))
throw new ArgumentException ("Generic format needs TableName input");
result = new GenericFile ().ProcessGenericFile (settings.InputFileName, client).Result;
break;
}
stopwatch.Stop ();
Logger.LogLine (LogLevel.Info, "\n Finished!! Processed {0} points (Success: {1}, Failed:{2}) in {3}", result.PointsFound, result.PointsProcessed, result.PointsFailed, stopwatch.Elapsed.ToString ());
}
catch (AggregateException e)
{
Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.InnerException.GetType ().Name, e.InnerException.Message, e.InnerException.StackTrace);
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
}
return (int) result.ExitCode;
}
public async Task TestGetInfluxDBNamesAsync_ServiceUnavailable()
{
var client = new InfluxDBClient (invalidInfluxUrl);
var r = await client.GetInfluxDBNamesAsync ();
}
public async Task TestPostPointsAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var today = DateTime.Now.ToShortDateString ();
var now = DateTime.Now.ToShortTimeString ();
var points = new List<IInfluxDatapoint> ();
var valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ());
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Nanoseconds;
points.Add (valDouble);
valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", DataGen.RandomDouble ());
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Microseconds;
points.Add (valDouble);
var valInt = new InfluxDatapoint<int> ();
valInt.UtcTimestamp = DateTime.UtcNow;
valInt.Tags.Add ("TestDate", today);
valInt.Tags.Add ("TestTime", now);
valInt.Fields.Add ("Intfield", DataGen.RandomInt ());
valInt.Fields.Add ("Intfield2", DataGen.RandomInt ());
valInt.MeasurementName = measurementName;
valInt.Precision = TimePrecision.Milliseconds;
points.Add (valInt);
valInt = new InfluxDatapoint<int> ();
valInt.UtcTimestamp = DateTime.UtcNow;
valInt.Tags.Add ("TestDate", today);
valInt.Tags.Add ("TestTime", now);
valInt.Fields.Add ("Intfield", DataGen.RandomInt ());
valInt.Fields.Add ("Intfield2", DataGen.RandomInt ());
valInt.MeasurementName = measurementName;
valInt.Precision = TimePrecision.Seconds;
points.Add (valInt);
var valBool = new InfluxDatapoint<bool> ();
valBool.UtcTimestamp = DateTime.UtcNow;
valBool.Tags.Add ("TestDate", today);
valBool.Tags.Add ("TestTime", now);
valBool.Fields.Add ("Booleanfield", time.Ticks % 2 == 0);
valBool.MeasurementName = measurementName;
valBool.Precision = TimePrecision.Minutes;
points.Add (valBool);
var valString = new InfluxDatapoint<string> ();
valString.UtcTimestamp = DateTime.UtcNow;
valString.Tags.Add ("TestDate", today);
valString.Tags.Add ("TestTime", now);
valString.Fields.Add ("Stringfield", DataGen.RandomString ());
valString.MeasurementName = measurementName;
valString.Precision = TimePrecision.Hours;
points.Add (valString);
var r = await client.PostPointsAsync (dbName, points);
Assert.IsTrue (r, "PostPointsAsync retunred false");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestCreateDatabaseAsync_InvalidName()
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.CreateDatabaseAsync (invalidDbName);
}
public async Task TestPostPointAsync_InvalidReq()
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var today = DateTime.Now.ToShortDateString ();
var now = DateTime.Now.ToShortTimeString ();
var valDouble = new InfluxDatapoint<double> ();
valDouble.UtcTimestamp = DateTime.UtcNow;
valDouble.Tags.Add ("TestDate", today);
valDouble.Tags.Add ("TestTime", now);
valDouble.Fields.Add ("Doublefield", DataGen.RandomDouble ());
valDouble.Fields.Add ("Doublefield2", Double.NaN);
valDouble.MeasurementName = measurementName;
valDouble.Precision = TimePrecision.Seconds;
var r = await client.PostPointAsync (dbName, valDouble);
Assert.IsTrue (r, "PostPointsAsync retunred false");
}
public async Task TestCreateDatabaseAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.CreateDatabaseAsync (dbName);
Assert.IsTrue (r, "CreateDatabaseAsync retunred false");
}
catch ( InvalidOperationException e )
{
StringAssert.Equals (e.Message, "database already exists");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestGetInfluxDBNamesAsync_Auth2()
{
var client = new InfluxDBClient (influxUrl, invalidUName, dbpwd);
var r = await client.GetInfluxDBNamesAsync ();
}
public async Task TestQueryAsync()
{
try
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.QueryDBAsync ("stress", "select * from performance limit 10");
DateTime d;
Assert.IsTrue (r != null && DateTime.TryParse (r[0].time, out d), "QueryDBAsync retunred null or invalid data");
}
catch ( Exception e )
{
Assert.Fail ("Unexpected exception of type {0} caught: {1}",
e.GetType (), e.Message);
return;
}
}
public async Task TestQueryAsync_MultiSeries()
{
var client = new InfluxDBClient (influxUrl, dbUName, dbpwd);
var r = await client.QueryDBAsync ("_internal", "Show Series");
}
public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus ();
int linesProcessed = 0;
int failedLines = 0;
int failedReqCount = 0;
try
{
var failureReasons = new Dictionary<Type, FailureTracker> ();
Stopwatch stopwatch = new Stopwatch ();
stopwatch.Start ();
var firstLine = File.ReadLines (InputFileName).FirstOrDefault ();
var firstCol = firstLine.Substring (0, firstLine.IndexOf (','));
if (!firstCol.Contains ("PDH-CSV"))
throw new InvalidDataException ("Input file is not a Standard Perfmon csv file");
var x = Regex.Matches (firstCol, "([-0-9]+)");
if (x.Count > 0)
minOffset = int.Parse (x[3].ToString ());
//get the column headers
List<PerfmonCounter> pecrfCounters;
try
{
pecrfCounters = ParsePerfMonFileHeader (firstLine);
}
catch (Exception ex)
{
throw new InvalidDataException ("Unable to parse file headers", ex);
}
IInfluxDatabase dbStructure;
IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup;
if (settings.PerfmonFile.Filter != Filters.None)
{
var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false);
dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject);
}
else
{
perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject);
}
List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> ();
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy ()
{
Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync (policy))
throw new InvalidOperationException ("Unable to create retention policy");
}
}
else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
//Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
foreach (var line in File.ReadLines (InputFileName).Skip (1))
{
try
{
var linePoints = ProcessPerfmonLogLine (line, perfGroup);
linesProcessed++;
if (linePoints == null || linePoints.Count == 0)
failedLines++;
else
{
result.PointsFound += linePoints.Count;
if (points == null)
points = linePoints;
else
points.AddRange (linePoints);
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count;
}
else
{
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 4)
break;
}
//a point will be either posted to Influx or in retry queue
points = null;
}
}
}
catch (Exception e)
{
failedLines++;
var type = e.GetType ();
if (!failureReasons.ContainsKey (type))
failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
failureReasons[type].LineNumbers.Add (linesProcessed);
}
if (failedLines > 0 || retryQueue.Count > 0)
Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
else
Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound);
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear ();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange (points.Where (p => p.Saved != true));
result.PointsProcessed += points.Count (p => p.Saved);
}
}
if (retryQueue.Count > 0)
{
Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear ();
}
else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
else
{
result.PointsFailed += retryQueue.Count;
}
}
pecrfCounters.Clear ();
stopwatch.Stop ();
if (failedLines > 0 || retryQueue.Count > 0)
{
Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
foreach (var f in failureReasons.Values)
Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message);
if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count)
result.ExitCode = ExitCode.UnableToProcess;
else
result.ExitCode = ExitCode.ProcessedWithErrors;
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnknownError;
}
return result;
}
