Inheritance: IInfluxRetentionPolicy
コード例 #1
0
        /// <summary>
        /// Gets the list of retention policies present in a DB
        /// </summary>
        /// <param name="dbName">Name of the database</param>
        /// <returns>List of InfluxRetentionPolicy objects</returns>
        public async Task <List <IInfluxRetentionPolicy> > GetRetentionPoliciesAsync(string dbName)
        {
            var rawpolicies = await QueryMultiSeriesAsync(dbName, "show retention policies on " + dbName);

            var policies = new List <IInfluxRetentionPolicy>();

            foreach (var policy in rawpolicies.FirstOrDefault()?.Entries)
            {
                var pol = new InfluxRetentionPolicy()
                {
                    DBName    = dbName,
                    Name      = policy.Name,
                    Duration  = StringHelper.ParseDuration(policy.Duration),
                    IsDefault = (policy.Default == "true"),
                    ReplicaN  = int.Parse(policy.ReplicaN),
                    Saved     = true
                };
                try
                {
                    //supported from Influx 12 onwards
                    pol.ShardDuration = StringHelper.ParseDuration(policy.ShardGroupDuration);
                }
                catch (Exception) { }
                policies.Add(pol);
            }
            return(policies);
        }
コード例 #2
0
ファイル: GenericFile.cs プロジェクト: AdysTech/Influxer
        public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client)
        {
            ProcessStatus result = new ProcessStatus ();

            int failedReqCount = 0;

            try
            {
                Stopwatch stopwatch = new Stopwatch ();
                stopwatch.Start ();

                var r = GetFileLayout (InputFileName);
                if (r.ExitCode != ExitCode.Success)
                    return r;

                IInfluxDatabase dbStructure;
                if (settings.GenericFile.Filter != Filters.None)
                {
                    var filterColumns = new List<GenericColumn> ();
                    if (settings.GenericFile.Filter == Filters.Columns)
                    {
                        if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
                            Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
                        else
                            filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ());
                    }

                    dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
                    ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);

                }

                var validity = ValidateData (InputFileName);

                var failureReasons = new Dictionary<Type, FailureTracker> ();

                List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> ();
                IInfluxRetentionPolicy policy = null;

                if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                {
                    var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
                    //if duraiton is specified that takes precidence
                    if (settings.InfluxDB.RetentionDuration != 0)
                    {
                        policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);

                        if (policy == null)
                        {
                            policy = new InfluxRetentionPolicy ()
                            {
                                Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
                                DBName = settings.InfluxDB.DatabaseName,
                                Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
                                IsDefault = false,
                                ReplicaN = 1
                            };
                            if (!await client.CreateRetentionPolicyAsync (policy))
                                throw new InvalidOperationException ("Unable to create retention policy");
                        }
                    }
                    else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                    {
                        policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
                        if (policy == null)
                            throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
                    }
                }


                foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
                {
                    if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker)))
                        continue;

                    try
                    {
                        result.PointsFound++;
                        var point = ProcessGenericLine (line, ColumnHeaders);
                        if (point == null)
                            result.PointsFailed++;
                        else
                        {
                            point.Retention = policy;
                            points.Add (point);
                        }

                        if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
                        {
                            bool postresult = false;
                            try
                            {
                                postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
                            }
                            catch (ServiceUnavailableException)
                            {
                                postresult = false;
                            }

                            if (postresult)
                            {
                                failedReqCount = 0;
                                result.PointsProcessed += points.Count;
                            }
                            else
                            {
                                //add failed to retry queue
                                retryQueue.AddRange (points.Where (p => p.Saved != true));
                                result.PointsProcessed += points.Count (p => p.Saved);
                                //avoid failing on too many points
                                if (++failedReqCount > 3)
                                    break;
                            }
                            //a point will be either posted to Influx or in retry queue
                            points.Clear ();
                        }
                    }
                    catch (Exception e)
                    {
                        result.PointsFailed++;
                        var type = e.GetType ();
                        if (!failureReasons.ContainsKey (type))
                            failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
                        failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);

                        //avoid too many failures, may be config is wrong
                        if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
                        {
                            Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
                            Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
                            break;
                        }
                    }

                    if (result.PointsFailed > 0 || retryQueue.Count > 0)
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3}                        ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
                    else
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}                          ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound);

                }

                //if we reached here due to repeated failures
                if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
                    throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");


                //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
                if (points != null && points.Count > 0)
                {

                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
                    {
                        result.PointsProcessed += points.Count;
                        points.Clear ();
                    }
                    else
                    {
                        failedReqCount++;
                        //add failed to retry queue
                        retryQueue.AddRange (points.Where (p => p.Saved != true));
                        result.PointsProcessed += points.Count (p => p.Saved);
                    }
                }

                //retry all previously failed points
                if (retryQueue.Count > 0)
                {
                    Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
                    {
                        result.PointsProcessed += retryQueue.Count;
                        retryQueue.Clear ();
                    }
                    else
                    {
                        result.PointsFailed += retryQueue.Count;
                        if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
                            throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                    }
                }

                stopwatch.Stop ();
                if (result.PointsFailed > 0)
                {
                    Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
                    foreach (var f in failureReasons.Values)
                        Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message);
                    if (result.PointsFailed == result.PointsFound)
                        result.ExitCode = ExitCode.UnableToProcess;
                    else
                        result.ExitCode = ExitCode.ProcessedWithErrors;
                }
                else
                {
                    result.ExitCode = ExitCode.Success;
                    Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
                }

            }
            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
                Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
                result.ExitCode = ExitCode.UnableToProcess;
            }
            return result;
        }
コード例 #3
0
        /// <summary>
        /// Gets the list of retention policies present in a DB
        /// </summary>
        /// <param name="dbName">Name of the database</param>
        /// <returns>List of InfluxRetentionPolicy objects</returns>
        public async Task<List<IInfluxRetentionPolicy>> GetRetentionPoliciesAsync (string dbName)
        {
            var rawpolicies = await QueryMultiSeriesAsync (dbName, "show retention policies on " + dbName);
            var policies = new List<IInfluxRetentionPolicy> ();

            foreach (var policy in rawpolicies.FirstOrDefault ()?.Entries)
            {
                var pol = new InfluxRetentionPolicy ()
                {
                    DBName = dbName,
                    Name = policy.Name,
                    Duration = StringHelper.ParseDuration (policy.Duration),
                    IsDefault = (policy.Default == "true"),
                    ReplicaN = int.Parse (policy.ReplicaN),
                    Saved = true
                };
                try
                {
                    //supported from Influx 12 onwards
                    pol.ShardDuration = StringHelper.ParseDuration (policy.ShardGroupDuration);
                }
                catch (Exception) { }
                policies.Add (pol);
            }
            return policies;

        }