/// <summary> /// Gets the list of retention policies present in a DB /// </summary> /// <param name="dbName">Name of the database</param> /// <returns>List of InfluxRetentionPolicy objects</returns> public async Task <List <IInfluxRetentionPolicy> > GetRetentionPoliciesAsync(string dbName) { var rawpolicies = await QueryMultiSeriesAsync(dbName, "show retention policies on " + dbName); var policies = new List <IInfluxRetentionPolicy>(); foreach (var policy in rawpolicies.FirstOrDefault()?.Entries) { var pol = new InfluxRetentionPolicy() { DBName = dbName, Name = policy.Name, Duration = StringHelper.ParseDuration(policy.Duration), IsDefault = (policy.Default == "true"), ReplicaN = int.Parse(policy.ReplicaN), Saved = true }; try { //supported from Influx 12 onwards pol.ShardDuration = StringHelper.ParseDuration(policy.ShardGroupDuration); } catch (Exception) { } policies.Add(pol); } return(policies); }
public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client) { ProcessStatus result = new ProcessStatus (); int failedReqCount = 0; try { Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); var r = GetFileLayout (InputFileName); if (r.ExitCode != ExitCode.Success) return r; IInfluxDatabase dbStructure; if (settings.GenericFile.Filter != Filters.None) { var filterColumns = new List<GenericColumn> (); if (settings.GenericFile.Filter == Filters.Columns) { if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0) Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them"); else filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ()); } dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName); ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase); } var validity = ValidateData (InputFileName); var failureReasons = new Dictionary<Type, FailureTracker> (); List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> (); IInfluxRetentionPolicy policy = null; if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName); //if duraiton is specified that takes precidence if (settings.InfluxDB.RetentionDuration != 0) { policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration); if (policy == null) { policy = new InfluxRetentionPolicy () { Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy, DBName = settings.InfluxDB.DatabaseName, Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration), IsDefault = false, ReplicaN = 1 }; if (!await client.CreateRetentionPolicyAsync (policy)) throw new InvalidOperationException ("Unable to create retention policy"); } } else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy)) { policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy); if (policy == null) throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy); } } foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows)) { if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker))) continue; try { result.PointsFound++; var point = ProcessGenericLine (line, ColumnHeaders); if (point == null) result.PointsFailed++; else { point.Retention = policy; points.Add (point); } if (points.Count >= settings.InfluxDB.PointsInSingleBatch) { bool postresult = false; try { postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points); } catch (ServiceUnavailableException) { postresult = false; } if (postresult) { failedReqCount = 0; result.PointsProcessed += points.Count; } else { //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); //avoid failing on too many points if (++failedReqCount > 3) break; } //a point will be either posted to Influx or in retry queue points.Clear (); } } catch (Exception e) { result.PointsFailed++; var type = e.GetType (); if (!failureReasons.ContainsKey (type)) failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message }); failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1); //avoid too many failures, may be config is wrong if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3) { Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!"); Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!"); break; } } if (result.PointsFailed > 0 || retryQueue.Count > 0) Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count); else Logger.Log (LogLevel.Verbose, "\r{0} Processed {1} ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound); } //if we reached here due to repeated failures if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch) if (points != null && points.Count > 0) { if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points)) { result.PointsProcessed += points.Count; points.Clear (); } else { failedReqCount++; //add failed to retry queue retryQueue.AddRange (points.Where (p => p.Saved != true)); result.PointsProcessed += points.Count (p => p.Saved); } } //retry all previously failed points if (retryQueue.Count > 0) { Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count); if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue)) { result.PointsProcessed += retryQueue.Count; retryQueue.Clear (); } else { result.PointsFailed += retryQueue.Count; if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4) throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!"); } } stopwatch.Stop (); if (result.PointsFailed > 0) { Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed); foreach (var f in failureReasons.Values) Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message); if (result.PointsFailed == result.PointsFound) result.ExitCode = ExitCode.UnableToProcess; else result.ExitCode = ExitCode.ProcessedWithErrors; } else { result.ExitCode = ExitCode.Success; Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound); } } catch (Exception e) { Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName); Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace); result.ExitCode = ExitCode.UnableToProcess; } return result; }
/// <summary> /// Gets the list of retention policies present in a DB /// </summary> /// <param name="dbName">Name of the database</param> /// <returns>List of InfluxRetentionPolicy objects</returns> public async Task<List<IInfluxRetentionPolicy>> GetRetentionPoliciesAsync (string dbName) { var rawpolicies = await QueryMultiSeriesAsync (dbName, "show retention policies on " + dbName); var policies = new List<IInfluxRetentionPolicy> (); foreach (var policy in rawpolicies.FirstOrDefault ()?.Entries) { var pol = new InfluxRetentionPolicy () { DBName = dbName, Name = policy.Name, Duration = StringHelper.ParseDuration (policy.Duration), IsDefault = (policy.Default == "true"), ReplicaN = int.Parse (policy.ReplicaN), Saved = true }; try { //supported from Influx 12 onwards pol.ShardDuration = StringHelper.ParseDuration (policy.ShardGroupDuration); } catch (Exception) { } policies.Add (pol); } return policies; }