public async Task TestPostPointsAsync_FromObject_AutogenRetention()
{
try
{
var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
var retention = new InfluxRetentionPolicy()
{
Name = "autogen", DBName = dbName, Duration = TimeSpan.FromMinutes(0), IsDefault = true
};
var points = Enumerable.Range(0, 10).Select(i =>
new PointObjectWithObjectRetention
{
Measurement = "RetentionTest",
Time = DateTime.UtcNow.AddDays(-i),
Retention = retention,
Precision = TimePrecision.Milliseconds,
StringTag = "FromObject",
IntTag = DataGen.RandomInt(),
StringField = DataGen.RandomString(),
DoubleField = DataGen.RandomDouble(),
IntField = DataGen.RandomInt(),
BoolField = i % 2 == 0,
});
var r = await client.PostPointsAsync(dbName, points);
Assert.IsTrue(r, "PostPointsAsync returned false");
}
catch (Exception e)
{
Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}");
return;
}
}
public async Task InitializeAsync()
{
ORM orm = null;
using (Stream stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Orchestrator.Service.TelemetrySelection.json"))
using (StreamReader reader = new StreamReader(stream))
{
string result = reader.ReadToEnd();
orm = JsonConvert.DeserializeObject <ORM>(result, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
MissingMemberHandling = MissingMemberHandling.Error
});
}
while (true)
{
try
{
await InfluxDBClient.GetInfluxDBNamesAsync();
Log.Information("Connected to Influxdb");
break;
}
catch (Exception ex)
{
Log.Information("Could not connect to Influx: " + ex.Message);
Log.Information("Retying in 5 seconds...");
}
await Task.Delay(5000);
}
foreach (var db in orm.Databases)
{
if (string.IsNullOrWhiteSpace(db.Name))
{
throw new Exception("Error: Empty or null database name in TelemetrySelection: " + orm);
}
//can check if DB exists first so success true/false has meaning
//can add check for success = false if DB doesn't already exist
bool success = await InfluxDBClient.CreateDatabaseAsync(db.Name);
var retentionPolicy = new InfluxRetentionPolicy()
{
Name = "ConfiguredRetentionPolicyFromEnv",
DBName = db.Name,
Duration = TimeSpan.FromDays(Configuration.GetValue("INFLUX_RETENTION_IN_DAYS", 30)),
IsDefault = false
};
if (!await InfluxDBClient.CreateRetentionPolicyAsync(retentionPolicy))
{
throw new Exception("Error: Could not create retention policy for influxdb: " + db.Name);
}
}
databases = orm.Databases;
}
public async Task TestCreateRetentionPolicy()
{
var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
var p = new InfluxRetentionPolicy()
{
Name = "Test2", DBName = dbName, Duration = TimeSpan.FromMinutes(1150), IsDefault = false
};
var r = await client.CreateRetentionPolicyAsync(p);
Assert.IsTrue(p.Saved, "CreateRetentionPolicyAsync failed");
}
private static async Task CreateRetentionPolicyLongterm()
{
var policies = await influxClient.GetRetentionPoliciesAsync(influxDbNameLongterm);
if (policies == null || !policies.Any(p => p.Name == ret_policy_longterm))
{
var rp_month = new InfluxRetentionPolicy()
{
Name = ret_policy_longterm, DBName = influxDbNameLongterm, Duration = TimeSpan.FromDays(10950), IsDefault = true
};
if (!await influxClient.CreateRetentionPolicyAsync(rp_month))
{
throw new InvalidOperationException("Unable to create Retention Policy for month");
}
}
}
private static async Task CreateRetentionPolicy()
{
var policies = await influxClient.GetRetentionPoliciesAsync(influxDbName);
if (policies == null || !policies.Any(p => p.Name == ret_policy_point))
{
var rp_point = new InfluxRetentionPolicy()
{
Name = ret_policy_point, DBName = influxDbName, Duration = TimeSpan.FromDays(365), IsDefault = true
};
if (!await influxClient.CreateRetentionPolicyAsync(rp_point))
{
throw new InvalidOperationException("Unable to create Retention Policy for pint");
}
}
}
public async Task TestPostPointsAsync_AutogenRetention()
{
try
{
var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
var time = DateTime.Now;
var TestDate = time.ToShortDateString();
var TestTime = time.ToShortTimeString();
var points = new List <IInfluxDatapoint>();
var retention = new InfluxRetentionPolicy()
{
Name = "autogen", DBName = dbName, Duration = TimeSpan.FromMinutes(0), IsDefault = true
};
for (var i = 0; i < 10; i++)
{
await Task.Delay(1);
var point = new InfluxDatapoint <long>();
point.Retention = retention;
point.UtcTimestamp = DateTime.UtcNow.AddDays(-i);
point.MeasurementName = "RetentionTest";
point.Precision = TimePrecision.Nanoseconds;
point.Tags.Add("TestDate", TestDate);
point.Tags.Add("TestTime", TestTime);
point.Fields.Add("Val", i);
points.Add(point);
}
var r = await client.PostPointsAsync(dbName, points);
Assert.IsTrue(r, "PostPointsAsync retunred false");
Assert.IsTrue(points.Count(p => p.Saved) == 10, "PostPointsAsync failed with autogen default retention policy");
}
catch (Exception e)
{
Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}");
return;
}
}
/// <summary>
/// 创建数据库和对应的数据保留策略
/// </summary>
/// <param name="dbName"></param>
private async Task CreateDatabaseAndRetentionPolicyAsync(string dbName)
{
//新建数据库，如果数据库已存在也会成功
_logger.LogDebug($"检查数据库 {dbName}");
await influxClient.CreateDatabaseAsync(dbName);
//新建保留策略
string rpName = GenerateRpName(dbName);//生成约定的保留策略名称
_logger.LogDebug($"检查保留策略 {rpName}");
var rp = new InfluxRetentionPolicy()
{
Name = rpName,
DBName = dbName,
Duration = TimeSpan.FromHours(_influxRetentionHours),
IsDefault = true
};
if (!await influxClient.CreateRetentionPolicyAsync(rp))
{
throw new InvalidOperationException($"为 {dbName} 创建保留策略失败，策略名称：{rpName}，保留时长：{_influxRetentionHours}小时");
}
}
public async Task TestPostPointsAsync_OlderthanRetention()
{
try
{
var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
InfluxRetentionPolicy retention = new InfluxRetentionPolicy()
{
Duration = TimeSpan.FromHours(1)
};
var points = await CreateTestPoints("RetentionTest", 10, TimePrecision.Nanoseconds, retention);
var r = await client.PostPointsAsync(dbName, points);
Assert.IsTrue(r, "PostPointsAsync retunred false");
Assert.IsTrue(points.Count(p => p.Saved) == 1, "PostPointsAsync saved points older than retention policy");
}
catch (Exception e)
{
Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}");
return;
}
}
public bool EnsureRetentionPolicy(string databaseName, InfluxDbRetentionPolicyDefinition definition)
{
var rps = _influxDbClient.GetRetentionPoliciesAsync(databaseName)
.Result
.FirstOrDefault(x => x.Name.Equals(definition.Name, StringComparison.InvariantCulture));;
if (rps != null)
{
_logger.Debug("Retention policy {0} already exists. Nothing to do.", definition.Name);
return(false);
}
_logger.Debug("Retention policy {0} does not exists. RP will be created.", definition.Name);
var rp = new InfluxRetentionPolicy()
{
Name = definition.Name,
DBName = databaseName,
Duration = definition.Duration,
IsDefault = definition.IsDefault
};
return(_influxDbClient.CreateRetentionPolicyAsync(rp).Result);
}
public async Task TestPostPointsAsync_AutogenRetention()
{
try
{
var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
var retention = new InfluxRetentionPolicy()
{
Name = "autogen", DBName = dbName, Duration = TimeSpan.FromMinutes(0), IsDefault = true
};
var points = await CreateTestPoints("RetentionTest", 10, TimePrecision.Nanoseconds, retention);
var r = await client.PostPointsAsync(dbName, points);
Assert.IsTrue(r, "PostPointsAsync retunred false");
Assert.IsTrue(points.Count(p => p.Saved) == 10, "PostPointsAsync failed with autogen default retention policy");
}
catch (Exception e)
{
Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}");
return;
}
}
public async Task <ProcessStatus> ProcessGenericFile(string InputFileName, InfluxDBClient client)
{
ProcessStatus result = new ProcessStatus();
int failedReqCount = 0;
try
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var r = GetFileLayout(InputFileName);
if (r.ExitCode != ExitCode.Success)
{
return(r);
}
IInfluxDatabase dbStructure;
if (settings.GenericFile.Filter != Filters.None)
{
var filterColumns = new List <GenericColumn>();
if (settings.GenericFile.Filter == Filters.Columns)
{
if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
{
Logger.LogLine(LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
}
else
{
filterColumns = ParseGenericColumns(settings.GenericFile.ColumnsFilter.ToString());
}
}
dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName);
ColumnHeaders = FilterGenericColumns(ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);
}
var validity = ValidateData(InputFileName);
var failureReasons = new Dictionary <Type, FailureTracker>();
List <IInfluxDatapoint> points = new List <IInfluxDatapoint>(), retryQueue = new List <IInfluxDatapoint>();
IInfluxRetentionPolicy policy = null;
if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName);
//if duraiton is specified that takes precidence
if (settings.InfluxDB.RetentionDuration != 0)
{
policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);
if (policy == null)
{
policy = new InfluxRetentionPolicy()
{
Name = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
DBName = settings.InfluxDB.DatabaseName,
Duration = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration),
IsDefault = false,
ReplicaN = 1
};
if (!await client.CreateRetentionPolicyAsync(policy))
{
throw new InvalidOperationException("Unable to create retention policy");
}
}
}
else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
{
policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy);
if (policy == null)
{
throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
}
}
}
foreach (var line in File.ReadLines(InputFileName).Skip(settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
{
if (String.IsNullOrWhiteSpace(line) || (!String.IsNullOrEmpty(settings.GenericFile.CommentMarker) && line.StartsWith(settings.GenericFile.CommentMarker)))
{
continue;
}
try
{
result.PointsFound++;
var point = ProcessGenericLine(line, ColumnHeaders);
if (point == null)
{
result.PointsFailed++;
}
else
{
point.Retention = policy;
points.Add(point);
}
if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
{
bool postresult = false;
try
{
postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points);
}
catch (ServiceUnavailableException)
{
postresult = false;
}
if (postresult)
{
failedReqCount = 0;
result.PointsProcessed += points.Count(p => p.Saved);
}
else
{
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
//avoid failing on too many points
if (++failedReqCount > 3)
{
break;
}
}
//a point will be either posted to Influx or in retry queue
points.Clear();
}
}
catch (Exception e)
{
result.PointsFailed++;
var type = e.GetType();
if (e is InfluxDBException)
{
if ((e as InfluxDBException).Reason == "Partial Write")
{
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
points.Clear();
}
}
if (!failureReasons.ContainsKey(type))
{
failureReasons.Add(type, new FailureTracker()
{
ExceptionType = type, Message = e.Message
});
}
failureReasons[type].LineNumbers.Add(result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);
//avoid too many failures, may be config is wrong
if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
{
Logger.LogLine(LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
Logger.LogLine(LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
break;
}
}
if (result.PointsFailed > 0 || retryQueue.Count > 0)
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
}
else
{
Logger.Log(LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound);
}
}
//if we reached here due to repeated failures
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
//finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
if (points != null && points.Count > 0)
{
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points))
{
result.PointsProcessed += points.Count;
points.Clear();
}
else
{
failedReqCount++;
//add failed to retry queue
retryQueue.AddRange(points.Where(p => p.Saved != true));
result.PointsProcessed += points.Count(p => p.Saved);
}
}
//retry all previously failed points
if (retryQueue.Count > 0)
{
Logger.LogLine(LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count);
try
{
if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue))
{
result.PointsProcessed += retryQueue.Count;
retryQueue.Clear();
}
else
{
result.PointsFailed += retryQueue.Count;
if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
{
throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
}
}
}
catch (InfluxDBException e)
{
if (e.Reason == "Partial Write")
{
}
}
}
stopwatch.Stop();
if (result.PointsFailed > 0)
{
Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
foreach (var f in failureReasons.Values)
{
Logger.LogLine(LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers.Take(5)), f.ExceptionType, f.Message);
}
if (result.PointsFailed == result.PointsFound)
{
result.ExitCode = ExitCode.UnableToProcess;
}
else
{
result.ExitCode = ExitCode.ProcessedWithErrors;
}
}
else
{
result.ExitCode = ExitCode.Success;
Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
}
}
catch (Exception e)
{
Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName);
Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
result.ExitCode = ExitCode.UnableToProcess;
}
return(result);
}
private static async Task <List <IInfluxDatapoint> > CreateTestPoints(string MeasurementName, int size, TimePrecision?precision = null, InfluxRetentionPolicy retention = null)
{
var time = DateTime.Now;
var TestDate = time.ToShortDateString();
var TestTime = time.ToShortTimeString();
var points = new List <IInfluxDatapoint>();
for (var i = 0; i < size; i++)
{
await Task.Delay(1);
var point = new InfluxDatapoint <long>();
if (retention != null)
{
point.Retention = retention;
}
point.UtcTimestamp = DateTime.UtcNow.AddDays(-i);
point.MeasurementName = MeasurementName;
if (precision != null)
{
point.Precision = precision.Value;
}
point.Tags.Add("TestDate", TestDate);
point.Tags.Add("TestTime", TestTime);
point.Fields.Add("Val", i);
points.Add(point);
}
return(points);
}
