private InfluxDatapoint <InfluxValueField> ProcessGenericLine(string line, List <GenericColumn> columnHeaders)
{
var columns = pattern.Split(line);
var columnCount = columns.Count();
var content = columns[settings.GenericFile.TimeColumn - 1].Replace("\"", "");
InfluxDatapoint <InfluxValueField> point = new InfluxDatapoint <InfluxValueField>()
{
Precision = settings.GenericFile.Precision,
MeasurementName = settings.InfluxDB.Measurement
};
point.InitializeTags(defaultTags);
var pointData = new Dictionary <GenericColumn, string>();
foreach (var c in columnHeaders)
{
content = columns[c.ColumnIndex].Replace("\"", "");
if (c.HasAutoGenColumns)
{
pointData.AddRange(c.SplitData(content));
}
else
{
pointData.Add(c, content);
}
}
foreach (var d in pointData)
{
content = d.Value;
if (d.Key.HasTransformations && d.Key.CanTransform(content))
{
content = d.Key.Transform(d.Value);
}
if (String.IsNullOrWhiteSpace(content))
{
continue;
}
if (d.Key.ColumnIndex == settings.GenericFile.TimeColumn - 1)
{
point.UtcTimestamp = ParseTimestamp(content);
}
else
{
double value = double.NaN; bool boolVal = false;
if (d.Key.Type == ColumnDataType.NumericalField)
{
if (!Double.TryParse(content, out value) || double.IsNaN(value))
{
throw new InvalidDataException(d.Key.ColumnHeader + " has inconsistent data, Unable to parse \"" + content + "\" as number");
}
point.Fields.Add(d.Key.ColumnHeader, new InfluxValueField(Math.Round(value, 2)));
}
else if (d.Key.Type == ColumnDataType.StringField)
{
point.Fields.Add(d.Key.ColumnHeader, new InfluxValueField(content));
}
else if (d.Key.Type == ColumnDataType.BooleanField)
{
if (!Boolean.TryParse(content, out boolVal))
{
throw new InvalidDataException(d.Key.ColumnHeader + " has inconsistent data, Unable to parse \"" + content + "\" as Boolean");
}
point.Fields.Add(d.Key.ColumnHeader, new InfluxValueField(boolVal));
}
else if (d.Key.Type == ColumnDataType.Tag)
{
point.Tags.Add(d.Key.ColumnHeader, content.Replace(settings.InfluxDB.InfluxReserved.ReservedCharecters.ToCharArray(), settings.InfluxDB.InfluxReserved.ReplaceReservedWith));
}
}
}
if (point.Fields.Count == 0)
{
throw new InvalidDataException("No values found on the row to post to Influx");
}
return(point);
}
private List <IInfluxDatapoint> ProcessPerfmonLogLine(string line, IEnumerable <IGrouping <string, PerfmonCounter> > perfGroup)
{
var columns = pattern.Split(line.Replace("\"", ""));
var columnCount = columns.Count();
if (!DateTime.TryParseExact(columns[0], settings.PerfmonFile.TimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime timeStamp))
{
throw new FormatException("Couldn't parse " + columns[0] + " using format " + settings.PerfmonFile.TimeFormat + ", check -timeformat argument");
}
var utcTime = timeStamp.AddMinutes(minOffset);
var points = new List <IInfluxDatapoint>();
foreach (var performanceObject in perfGroup)
{
foreach (var hostGrp in performanceObject.GroupBy(p => p.Host))
{
if (settings.PerfmonFile.MultiMeasurements)
{
var point = new InfluxDatapoint <double>
{
Precision = TimePrecision.Milliseconds,
Retention = policy,
MeasurementName = performanceObject.Key,
UtcTimestamp = utcTime
};
if (defaultTags.Count > 0)
{
point.InitializeTags(defaultTags);
}
point.Tags.Add("Host", hostGrp.Key);
double value = 0.0;
foreach (var counter in hostGrp)
{
if (!String.IsNullOrWhiteSpace(columns[counter.ColumnIndex]) && Double.TryParse(columns[counter.ColumnIndex], out value))
{
//Perfmon file can have duplicate columns!!
if (point.Fields.ContainsKey(counter.CounterName))
{
point.Fields[counter.CounterName] = value;
}
else
{
point.Fields.Add(counter.CounterName, value);
}
}
}
if (point.Fields.Count > 0)
{
points.Add(point);
}
}
else
{
foreach (var counter in hostGrp)
{
if (!String.IsNullOrWhiteSpace(columns[counter.ColumnIndex]) && Double.TryParse(columns[counter.ColumnIndex], out double value))
{
var point = new InfluxDatapoint <double>()
{
Precision = TimePrecision.Milliseconds,
Retention = policy,
MeasurementName = settings.InfluxDB.Measurement,
UtcTimestamp = utcTime
};
if (defaultTags.Count > 0)
{
point.InitializeTags(defaultTags);
}
point.Tags.Add("Host", hostGrp.Key);
point.Tags.Add("PerformanceObject", counter.PerformanceObject);
point.Tags.Add("PerformanceCounter", counter.CounterName);
point.Fields.Add("CounterValue", value);
points.Add(point);
}
}
}
}
}
return(points);
}
private InfluxDatapoint<InfluxValueField> ProcessGenericLine (string line, List<GenericColumn> columnHeaders)
{
var columns = pattern.Split (line);
var columnCount = columns.Count ();
var content = columns[settings.GenericFile.TimeColumn - 1].Replace ("\"", "");
InfluxDatapoint<InfluxValueField> point = new InfluxDatapoint<InfluxValueField> ();
point.Precision = settings.GenericFile.Precision;
point.MeasurementName = settings.InfluxDB.Measurement;
point.InitializeTags (defaultTags);
var pointData = new Dictionary<GenericColumn, string> ();
foreach (var c in columnHeaders)
{
content = columns[c.ColumnIndex].Replace ("\"", "");
if (c.HasAutoGenColumns)
{
pointData.AddRange (c.SplitData (content));
}
else
{
pointData.Add (c, content);
}
}
foreach (var d in pointData)
{
content = d.Value;
if (d.Key.HasTransformations && d.Key.CanTransform (content))
content = d.Key.Transform (d.Value);
if (String.IsNullOrWhiteSpace (content)) continue;
if (d.Key.ColumnIndex == settings.GenericFile.TimeColumn - 1)
{
DateTime timeStamp;
if (!DateTime.TryParseExact (content, settings.GenericFile.TimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out timeStamp))
throw new FormatException ("Couldn't parse " + content + " using format " + settings.GenericFile.TimeFormat + ", check -timeformat argument");
point.UtcTimestamp = timeStamp.AddMinutes (settings.GenericFile.UtcOffset);
}
else
{
double value = double.NaN; bool boolVal = false;
if (d.Key.Type == ColumnDataType.NumericalField)
{
if (!Double.TryParse (content, out value) || double.IsNaN (value))
throw new InvalidDataException (d.Key.ColumnHeader + " has inconsistent data, Unable to parse \"" + content + "\" as number");
point.Fields.Add (d.Key.ColumnHeader, new InfluxValueField (Math.Round (value, 2)));
}
else if (d.Key.Type == ColumnDataType.StringField)
{
point.Fields.Add (d.Key.ColumnHeader, new InfluxValueField (content));
}
else if (d.Key.Type == ColumnDataType.BooleanField)
{
if (!Boolean.TryParse (content, out boolVal))
throw new InvalidDataException (d.Key.ColumnHeader + " has inconsistent data, Unable to parse \"" + content + "\" as Boolean");
point.Fields.Add (d.Key.ColumnHeader, new InfluxValueField (boolVal));
}
else if (d.Key.Type == ColumnDataType.Tag)
point.Tags.Add (d.Key.ColumnHeader, content.Replace (settings.InfluxDB.InfluxReserved.ReservedCharecters.ToCharArray (), settings.InfluxDB.InfluxReserved.ReplaceReservedWith));
}
}
if (point.Fields.Count == 0)
throw new InvalidDataException ("No values found on the row to post to Influx");
return point;
}
private List<IInfluxDatapoint> ProcessPerfmonLogLine (string line, IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup)
{
var columns = pattern.Split (line.Replace ("\"", ""));
var columnCount = columns.Count ();
DateTime timeStamp;
if (!DateTime.TryParseExact (columns[0], settings.PerfmonFile.TimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out timeStamp))
throw new FormatException ("Couldn't parse " + columns[0] + " using format " + settings.PerfmonFile.TimeFormat + ", check -timeformat argument");
var utcTime = timeStamp.AddMinutes (minOffset);
var points = new List<IInfluxDatapoint> ();
foreach (var performanceObject in perfGroup)
{
foreach (var hostGrp in performanceObject.GroupBy (p => p.Host))
{
if (settings.PerfmonFile.MultiMeasurements)
{
var point = new InfluxDatapoint<double> ();
if (defaultTags.Count > 0) point.InitializeTags (defaultTags);
point.Tags.Add ("Host", hostGrp.Key);
point.Precision = TimePrecision.Seconds;
point.Retention = policy;
point.MeasurementName = performanceObject.Key;
point.UtcTimestamp = utcTime;
double value = 0.0;
foreach (var counter in hostGrp)
{
if (!String.IsNullOrWhiteSpace (columns[counter.ColumnIndex]) && Double.TryParse (columns[counter.ColumnIndex], out value))
{
//Perfmon file can have duplicate columns!!
if (point.Fields.ContainsKey (counter.CounterName))
point.Fields[counter.CounterName] = value;
else
point.Fields.Add (counter.CounterName, value);
}
}
if (point.Fields.Count > 0)
points.Add (point);
}
else
{
foreach (var counter in hostGrp)
{
double value = 0.0;
if (!String.IsNullOrWhiteSpace (columns[counter.ColumnIndex]) && Double.TryParse (columns[counter.ColumnIndex], out value))
{
var point = new InfluxDatapoint<double> ();
point.Precision = TimePrecision.Seconds;
point.Retention = policy;
if (defaultTags.Count > 0) point.InitializeTags (defaultTags);
point.Tags.Add ("Host", hostGrp.Key);
point.MeasurementName = settings.InfluxDB.Measurement;
point.UtcTimestamp = utcTime;
point.Tags.Add ("PerformanceObject", counter.PerformanceObject);
point.Tags.Add ("PerformanceCounter", counter.CounterName);
point.Fields.Add ("CounterValue", value);
points.Add (point);
}
}
}
}
}
return points;
}
