示例#1
0
        public async Task <ProcessStatus> ProcessGenericFile(string InputFileName, InfluxDBClient client)
        {
            ProcessStatus result = new ProcessStatus();

            int failedReqCount = 0;

            try
            {
                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();

                var r = GetFileLayout(InputFileName);
                if (r.ExitCode != ExitCode.Success)
                {
                    return(r);
                }

                IInfluxDatabase dbStructure;
                if (settings.GenericFile.Filter != Filters.None)
                {
                    var filterColumns = new List <GenericColumn>();
                    if (settings.GenericFile.Filter == Filters.Columns)
                    {
                        if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
                        {
                            Logger.LogLine(LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
                        }
                        else
                        {
                            filterColumns = ParseGenericColumns(settings.GenericFile.ColumnsFilter.ToString());
                        }
                    }

                    dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName);

                    ColumnHeaders = FilterGenericColumns(ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);
                }

                var validity = ValidateData(InputFileName);

                var failureReasons = new Dictionary <Type, FailureTracker>();

                List <IInfluxDatapoint> points = new List <IInfluxDatapoint>(), retryQueue = new List <IInfluxDatapoint>();
                IInfluxRetentionPolicy  policy = null;

                if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
                {
                    var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName);

                    //if duraiton is specified that takes precidence
                    if (settings.InfluxDB.RetentionDuration != 0)
                    {
                        policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);

                        if (policy == null)
                        {
                            policy = new InfluxRetentionPolicy()
                            {
                                Name      = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
                                DBName    = settings.InfluxDB.DatabaseName,
                                Duration  = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration),
                                IsDefault = false,
                                ReplicaN  = 1
                            };
                            if (!await client.CreateRetentionPolicyAsync(policy))
                            {
                                throw new InvalidOperationException("Unable to create retention policy");
                            }
                        }
                    }
                    else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
                    {
                        policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy);
                        if (policy == null)
                        {
                            throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
                        }
                    }
                }

                foreach (var line in File.ReadLines(InputFileName).Skip(settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
                {
                    if (String.IsNullOrWhiteSpace(line) || (!String.IsNullOrEmpty(settings.GenericFile.CommentMarker) && line.StartsWith(settings.GenericFile.CommentMarker)))
                    {
                        continue;
                    }
                    try
                    {
                        result.PointsFound++;
                        var point = ProcessGenericLine(line, ColumnHeaders);
                        if (point == null)
                        {
                            result.PointsFailed++;
                        }
                        else
                        {
                            point.Retention = policy;
                            points.Add(point);
                        }

                        if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
                        {
                            bool postresult = false;
                            try
                            {
                                postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points);
                            }
                            catch (ServiceUnavailableException)
                            {
                                postresult = false;
                            }

                            if (postresult)
                            {
                                failedReqCount          = 0;
                                result.PointsProcessed += points.Count(p => p.Saved);
                            }
                            else
                            {
                                //add failed to retry queue
                                retryQueue.AddRange(points.Where(p => p.Saved != true));
                                result.PointsProcessed += points.Count(p => p.Saved);
                                //avoid failing on too many points
                                if (++failedReqCount > 3)
                                {
                                    break;
                                }
                            }
                            //a point will be either posted to Influx or in retry queue
                            points.Clear();
                        }
                    }
                    catch (Exception e)
                    {
                        result.PointsFailed++;
                        var type = e.GetType();
                        if (e is InfluxDBException)
                        {
                            if ((e as InfluxDBException).Reason == "Partial Write")
                            {
                                retryQueue.AddRange(points.Where(p => p.Saved != true));
                                result.PointsProcessed += points.Count(p => p.Saved);
                                points.Clear();
                            }
                        }
                        if (!failureReasons.ContainsKey(type))
                        {
                            failureReasons.Add(type, new FailureTracker()
                            {
                                ExceptionType = type, Message = e.Message
                            });
                        }
                        failureReasons[type].LineNumbers.Add(result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);

                        //avoid too many failures, may be config is wrong
                        if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
                        {
                            Logger.LogLine(LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
                            Logger.LogLine(LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
                            break;
                        }
                    }

                    if (result.PointsFailed > 0 || retryQueue.Count > 0)
                    {
                        Logger.Log(LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3}                        ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
                    }
                    else
                    {
                        Logger.Log(LogLevel.Verbose, "\r{0} Processed {1}                          ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), result.PointsFound);
                    }
                }

                //if we reached here due to repeated failures
                if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
                {
                    throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                }

                //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
                if (points != null && points.Count > 0)
                {
                    if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points))
                    {
                        result.PointsProcessed += points.Count;
                        points.Clear();
                    }
                    else
                    {
                        failedReqCount++;
                        //add failed to retry queue
                        retryQueue.AddRange(points.Where(p => p.Saved != true));
                        result.PointsProcessed += points.Count(p => p.Saved);
                    }
                }

                //retry all previously failed points
                if (retryQueue.Count > 0)
                {
                    Logger.LogLine(LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count);
                    try
                    {
                        if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue))
                        {
                            result.PointsProcessed += retryQueue.Count;
                            retryQueue.Clear();
                        }
                        else
                        {
                            result.PointsFailed += retryQueue.Count;
                            if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
                            {
                                throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                            }
                        }
                    }
                    catch (InfluxDBException e)
                    {
                        if (e.Reason == "Partial Write")
                        {
                        }
                    }
                }

                stopwatch.Stop();
                if (result.PointsFailed > 0)
                {
                    Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
                    foreach (var f in failureReasons.Values)
                    {
                        Logger.LogLine(LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers.Take(5)), f.ExceptionType, f.Message);
                    }
                    if (result.PointsFailed == result.PointsFound)
                    {
                        result.ExitCode = ExitCode.UnableToProcess;
                    }
                    else
                    {
                        result.ExitCode = ExitCode.ProcessedWithErrors;
                    }
                }
                else
                {
                    result.ExitCode = ExitCode.Success;
                    Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
                }
            }
            catch (Exception e)
            {
                Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName);
                Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
                result.ExitCode = ExitCode.UnableToProcess;
            }
            return(result);
        }
示例#2
0
        public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, InfluxDBClient client)
        {
            ProcessStatus result = new ProcessStatus ();

            int failedReqCount = 0;

            try
            {
                Stopwatch stopwatch = new Stopwatch ();
                stopwatch.Start ();

                var r = GetFileLayout (InputFileName);
                if (r.ExitCode != ExitCode.Success)
                    return r;

                IInfluxDatabase dbStructure;
                if (settings.GenericFile.Filter != Filters.None)
                {
                    var filterColumns = new List<GenericColumn> ();
                    if (settings.GenericFile.Filter == Filters.Columns)
                    {
                        if (settings.GenericFile.ColumnLayout != null && settings.GenericFile.ColumnLayout.Count > 0)
                            Logger.LogLine (LogLevel.Info, "Column Filtering is not applicable when columns are defined in Config file. Use the Skip attribute on each column to filter them");
                        else
                            filterColumns = ParseGenericColumns (settings.GenericFile.ColumnsFilter.Columns.ToString ());
                    }

                    dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
                    ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);

                }

                var validity = ValidateData (InputFileName);

                var failureReasons = new Dictionary<Type, FailureTracker> ();

                List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> ();
                IInfluxRetentionPolicy policy = null;

                if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                {
                    var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
                    //if duraiton is specified that takes precidence
                    if (settings.InfluxDB.RetentionDuration != 0)
                    {
                        policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);

                        if (policy == null)
                        {
                            policy = new InfluxRetentionPolicy ()
                            {
                                Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
                                DBName = settings.InfluxDB.DatabaseName,
                                Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
                                IsDefault = false,
                                ReplicaN = 1
                            };
                            if (!await client.CreateRetentionPolicyAsync (policy))
                                throw new InvalidOperationException ("Unable to create retention policy");
                        }
                    }
                    else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                    {
                        policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
                        if (policy == null)
                            throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
                    }
                }


                foreach (var line in File.ReadLines (InputFileName).Skip (settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows))
                {
                    if (String.IsNullOrWhiteSpace (line) || (!String.IsNullOrEmpty (settings.GenericFile.CommentMarker) && line.StartsWith (settings.GenericFile.CommentMarker)))
                        continue;

                    try
                    {
                        result.PointsFound++;
                        var point = ProcessGenericLine (line, ColumnHeaders);
                        if (point == null)
                            result.PointsFailed++;
                        else
                        {
                            point.Retention = policy;
                            points.Add (point);
                        }

                        if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
                        {
                            bool postresult = false;
                            try
                            {
                                postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
                            }
                            catch (ServiceUnavailableException)
                            {
                                postresult = false;
                            }

                            if (postresult)
                            {
                                failedReqCount = 0;
                                result.PointsProcessed += points.Count;
                            }
                            else
                            {
                                //add failed to retry queue
                                retryQueue.AddRange (points.Where (p => p.Saved != true));
                                result.PointsProcessed += points.Count (p => p.Saved);
                                //avoid failing on too many points
                                if (++failedReqCount > 3)
                                    break;
                            }
                            //a point will be either posted to Influx or in retry queue
                            points.Clear ();
                        }
                    }
                    catch (Exception e)
                    {
                        result.PointsFailed++;
                        var type = e.GetType ();
                        if (!failureReasons.ContainsKey (type))
                            failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
                        failureReasons[type].LineNumbers.Add (result.PointsFound + settings.GenericFile.HeaderRow + settings.GenericFile.SkipRows + 1);

                        //avoid too many failures, may be config is wrong
                        if (!settings.GenericFile.IgnoreErrors && result.PointsFailed > settings.InfluxDB.PointsInSingleBatch * 3)
                        {
                            Logger.LogLine (LogLevel.Info, "\n Too many failed points, refer to error info. Aborting!!");
                            Logger.LogLine (LogLevel.Error, "\n Too many failed points, refer to error info. Aborting!!");
                            break;
                        }
                    }

                    if (result.PointsFailed > 0 || retryQueue.Count > 0)
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}, Failed {2}, Queued {3}                        ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound, result.PointsFailed, retryQueue.Count);
                    else
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed {1}                          ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), result.PointsFound);

                }

                //if we reached here due to repeated failures
                if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
                    throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");


                //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
                if (points != null && points.Count > 0)
                {

                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
                    {
                        result.PointsProcessed += points.Count;
                        points.Clear ();
                    }
                    else
                    {
                        failedReqCount++;
                        //add failed to retry queue
                        retryQueue.AddRange (points.Where (p => p.Saved != true));
                        result.PointsProcessed += points.Count (p => p.Saved);
                    }
                }

                //retry all previously failed points
                if (retryQueue.Count > 0)
                {
                    Logger.LogLine (LogLevel.Verbose, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
                    {
                        result.PointsProcessed += retryQueue.Count;
                        retryQueue.Clear ();
                    }
                    else
                    {
                        result.PointsFailed += retryQueue.Count;
                        if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
                            throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                    }
                }

                stopwatch.Stop ();
                if (result.PointsFailed > 0)
                {
                    Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed{2}, Failed:{3}", (DateTime.Now - stopwatch.Elapsed), InputFileName, result.PointsFound, result.PointsFailed);
                    foreach (var f in failureReasons.Values)
                        Logger.LogLine (LogLevel.Error, "{0} lines (e.g. {1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers.Take (5)), f.ExceptionType, f.Message);
                    if (result.PointsFailed == result.PointsFound)
                        result.ExitCode = ExitCode.UnableToProcess;
                    else
                        result.ExitCode = ExitCode.ProcessedWithErrors;
                }
                else
                {
                    result.ExitCode = ExitCode.Success;
                    Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} points", result.PointsFound);
                }

            }
            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
                Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
                result.ExitCode = ExitCode.UnableToProcess;
            }
            return result;
        }
示例#3
0
        private static int Main(string[] args)
        {
            try
            {
                if (!CommandLineProcessor.ProcessArguments(args))
                {
                    return((int)ExitCode.Success);
                }
                settings = CommandLineProcessor.Settings;
            }
            catch (ArgumentException e)
            {
                Logger.LogLine(LogLevel.Error, e.Message);
                return((int)ExitCode.InvalidArgument);
            }
            catch (FileLoadException e)
            {
                Logger.LogLine(LogLevel.Error, e.Message);
                Logger.LogLine(LogLevel.Info, "Problem loading config file, regenerate it with /config option");
                return((int)ExitCode.InvalidFilename);
            }
            catch (Exception e)
            {
                Logger.LogLine(LogLevel.Error, "Error processing arguments {0}: {1}", e.GetType().Name, e.Message);
                return((int)ExitCode.InvalidArgument);
            }

            #region Validate inputs

            if (String.IsNullOrWhiteSpace(settings.InputFileName))
            {
                Logger.LogLine(LogLevel.Error, "Input File Name is not specified!! Can't continue");
                return((int)ExitCode.InvalidArgument);
            }

            try
            {
                settings.InputFileName = Path.GetFullPath(settings.InputFileName);
            }
            catch (Exception e)
            {
                Logger.LogLine(LogLevel.Error, "Error with input file:{0},{1}", e.GetType().Name, e.Message);
                Logger.LogLine(LogLevel.Info, "Problem with inputfile name, check path");
                return((int)ExitCode.InvalidFilename);
            }

            if (String.IsNullOrWhiteSpace(settings.InfluxDB.InfluxUri))
            {
                Logger.LogLine(LogLevel.Error, "Influx DB Uri is not configured!!");
                return((int)ExitCode.InvalidArgument);
            }

            if (String.IsNullOrWhiteSpace(settings.InfluxDB.DatabaseName))
            {
                Logger.LogLine(LogLevel.Error, "Influx DB name is not configured!!");
                return((int)ExitCode.InvalidArgument);
            }

            #endregion Validate inputs

            ProcessStatus result = new ProcessStatus()
            {
                ExitCode = ExitCode.UnknownError
            };
            try
            {
                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();

                var client = new InfluxDBClient(settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);

                if (!VerifyDatabaseAsync(client, settings.InfluxDB.DatabaseName).Result)
                {
                    Logger.LogLine(LogLevel.Info, "Unable to create DB {0}", settings.InfluxDB.DatabaseName);
                    return((int)ExitCode.UnableToProcess);
                }
                switch (settings.FileFormat)
                {
                case FileFormats.Perfmon:
                    result = new PerfmonFile().ProcessPerfMonLog(settings.InputFileName, client).Result;
                    break;

                case FileFormats.Generic:
                    if (String.IsNullOrWhiteSpace(settings.InfluxDB.Measurement))
                    {
                        throw new ArgumentException("Generic format needs TableName input");
                    }
                    result = new GenericFile().ProcessGenericFile(settings.InputFileName, client).Result;
                    break;
                }

                stopwatch.Stop();
                Logger.LogLine(LogLevel.Info, "\n Finished!! Processed {0} points (Success: {1}, Failed:{2}) in {3}", result.PointsFound, result.PointsProcessed, result.PointsFailed, stopwatch.Elapsed.ToString());
            }
            catch (AggregateException e)
            {
                Logger.LogLine(LogLevel.Error, "Error!! {0}:{1} - {2}", e.InnerException.GetType().Name, e.InnerException.Message, e.InnerException.StackTrace);
            }
            catch (Exception e)
            {
                Logger.LogLine(LogLevel.Error, "Error!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
            }
            return((int)result.ExitCode);
        }
示例#4
0
        public async Task <ProcessStatus> ProcessPerfMonLog(string InputFileName, InfluxDBClient client)
        {
            ProcessStatus result         = new ProcessStatus();
            int           linesProcessed = 0;
            int           failedLines    = 0;

            int failedReqCount = 0;

            try
            {
                var failureReasons = new Dictionary <Type, FailureTracker>();

                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();

                var firstLine = File.ReadLines(InputFileName).FirstOrDefault();

                var firstCol = firstLine.Substring(0, firstLine.IndexOf(','));
                if (!firstCol.Contains("PDH-CSV"))
                {
                    throw new InvalidDataException("Input file is not a Standard Perfmon csv file");
                }
                var x = Regex.Matches(firstCol, "([-0-9]+)");
                if (x.Count > 0)
                {
                    minOffset = int.Parse(x[3].ToString());
                }

                //get the column headers
                List <PerfmonCounter> pecrfCounters;
                try
                {
                    pecrfCounters = ParsePerfMonFileHeader(firstLine);
                }
                catch (Exception ex)
                {
                    throw new InvalidDataException("Unable to parse file headers", ex);
                }

                IInfluxDatabase dbStructure;
                IEnumerable <IGrouping <string, PerfmonCounter> > perfGroup;
                if (settings.PerfmonFile.Filter != Filters.None)
                {
                    var filterColumns = ParsePerfMonFileHeader(settings.PerfmonFile.ColumnsFilter.ToString(), false);
                    dbStructure = await client.GetInfluxDBStructureAsync(settings.InfluxDB.DatabaseName);

                    perfGroup = FilterPerfmonLogColumns(pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy(p => p.PerformanceObject);
                }
                else
                {
                    perfGroup = pecrfCounters.GroupBy(p => p.PerformanceObject);
                }

                List <IInfluxDatapoint> points = null, retryQueue = new List <IInfluxDatapoint>();

                if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
                {
                    var policies = await client.GetRetentionPoliciesAsync(settings.InfluxDB.DatabaseName);

                    //if duraiton is specified that takes precidence
                    if (settings.InfluxDB.RetentionDuration != 0)
                    {
                        policy = policies.FirstOrDefault(p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);

                        if (policy == null)
                        {
                            policy = new InfluxRetentionPolicy()
                            {
                                Name      = String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
                                DBName    = settings.InfluxDB.DatabaseName,
                                Duration  = TimeSpan.FromMinutes(settings.InfluxDB.RetentionDuration),
                                IsDefault = false,
                                ReplicaN  = 1
                            };
                            if (!await client.CreateRetentionPolicyAsync(policy))
                            {
                                throw new InvalidOperationException("Unable to create retention policy");
                            }
                        }
                    }
                    else if (!String.IsNullOrWhiteSpace(settings.InfluxDB.RetentionPolicy))
                    {
                        policy = policies.FirstOrDefault(p => p.Name == settings.InfluxDB.RetentionPolicy);
                        if (policy == null)
                        {
                            throw new ArgumentException("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
                        }
                    }
                }
                //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
                foreach (var line in File.ReadLines(InputFileName).Skip(1))
                {
                    try
                    {
                        var linePoints = ProcessPerfmonLogLine(line, perfGroup);
                        linesProcessed++;

                        if (linePoints == null || linePoints.Count == 0)
                        {
                            failedLines++;
                        }
                        else
                        {
                            result.PointsFound += linePoints.Count;

                            if (points == null)
                            {
                                points = linePoints;
                            }
                            else
                            {
                                points.AddRange(linePoints);
                            }

                            if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
                            {
                                bool postresult = false;
                                try
                                {
                                    postresult = await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points);
                                }
                                catch (ServiceUnavailableException)
                                {
                                    postresult = false;
                                }
                                if (postresult)
                                {
                                    failedReqCount          = 0;
                                    result.PointsProcessed += points.Count(p => p.Saved);
                                }
                                else
                                {
                                    //add failed to retry queue
                                    retryQueue.AddRange(points.Where(p => p.Saved != true));
                                    result.PointsProcessed += points.Count(p => p.Saved);
                                    //avoid failing on too many points
                                    if (++failedReqCount > 4)
                                    {
                                        break;
                                    }
                                }
                                //a point will be either posted to Influx or in retry queue
                                points = null;
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        failedLines++;
                        var type = e.GetType();
                        if (!failureReasons.ContainsKey(type))
                        {
                            failureReasons.Add(type, new FailureTracker()
                            {
                                ExceptionType = type, Message = e.Message
                            });
                        }
                        failureReasons[type].LineNumbers.Add(linesProcessed);
                    }

                    if (failedLines > 0 || retryQueue.Count > 0)
                    {
                        Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points     ", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    }
                    else
                    {
                        Logger.Log(LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound);
                    }
                }

                //if we reached here due to repeated failures
                if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
                {
                    throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                }

                //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
                if (points != null && points.Count > 0)
                {
                    if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, points))
                    {
                        result.PointsProcessed += points.Count;
                        points.Clear();
                    }
                    else
                    {
                        failedReqCount++;
                        //add failed to retry queue
                        retryQueue.AddRange(points.Where(p => p.Saved != true));
                        result.PointsProcessed += points.Count(p => p.Saved);
                    }
                }

                if (retryQueue.Count > 0)
                {
                    Logger.LogLine(LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), retryQueue.Count);
                    if (await client.PostPointsAsync(settings.InfluxDB.DatabaseName, retryQueue))
                    {
                        result.PointsProcessed += retryQueue.Count;
                        retryQueue.Clear();
                    }
                    else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
                    {
                        throw new InvalidOperationException("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                    }
                    else
                    {
                        result.PointsFailed += retryQueue.Count;
                    }
                }

                pecrfCounters.Clear();
                stopwatch.Stop();
                if (failedLines > 0 || retryQueue.Count > 0)
                {
                    Logger.Log(LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    Logger.LogLine(LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    foreach (var f in failureReasons.Values)
                    {
                        Logger.LogLine(LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join(",", f.LineNumbers), f.ExceptionType, f.Message);
                    }
                    if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count)
                    {
                        result.ExitCode = ExitCode.UnableToProcess;
                    }
                    else
                    {
                        result.ExitCode = ExitCode.ProcessedWithErrors;
                    }
                }
                else
                {
                    result.ExitCode = ExitCode.Success;
                    Logger.LogLine(LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound);
                }
            }
            catch (Exception e)
            {
                Logger.LogLine(LogLevel.Error, "Failed to process {0}", InputFileName);
                Logger.LogLine(LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType().Name, e.Message, e.StackTrace);
                result.ExitCode = ExitCode.UnknownError;
            }

            return(result);
        }
示例#5
0
        static int Main (string[] args)
        {
            try
            {
                if (!CommandLineProcessor.ProcessArguments (args))
                    return (int) ExitCode.Success;
                settings = CommandLineProcessor.Settings;
            }
            catch (ArgumentException e)
            {
                Logger.LogLine (LogLevel.Error, e.Message);
                return (int) ExitCode.InvalidArgument;
            }
            catch (FileLoadException e)
            {
                Logger.LogLine (LogLevel.Error, e.Message);
                Logger.LogLine (LogLevel.Info, "Problem loading config file, regenerate it with /config option");
                return (int) ExitCode.InvalidFilename;
            }
            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Error processing arguments {0}: {1}", e.GetType ().Name, e.Message);
                return (int) ExitCode.InvalidArgument;
            }

            #region Validate inputs
            
            if (String.IsNullOrWhiteSpace (settings.InputFileName))
            {
                Logger.LogLine (LogLevel.Error, "Input File Name is not specified!! Can't continue");
                return (int) ExitCode.InvalidArgument;
            }

            try
            {
                settings.InputFileName = Path.GetFullPath (settings.InputFileName);
            }
            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Error with input file:{0},{1}", e.GetType ().Name, e.Message);
                Logger.LogLine (LogLevel.Info, "Problem with inputfile name, check path");
                return (int) ExitCode.InvalidFilename;
            }

            if (String.IsNullOrWhiteSpace (settings.InfluxDB.InfluxUri))
            {
                Logger.LogLine (LogLevel.Error, "Influx DB Uri is not configured!!");
                return (int) ExitCode.InvalidArgument;
            }

            if (String.IsNullOrWhiteSpace (settings.InfluxDB.DatabaseName))
            {
                Logger.LogLine (LogLevel.Error, "Influx DB name is not configured!!");
                return (int) ExitCode.InvalidArgument;
            }

            #endregion

            ProcessStatus result = new ProcessStatus () { ExitCode = ExitCode.UnknownError };
            try
            {
                Stopwatch stopwatch = new Stopwatch ();
                stopwatch.Start ();

                var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);

                if (!VerifyDatabaseAsync (client, settings.InfluxDB.DatabaseName).Result)
                {
                    Logger.LogLine (LogLevel.Info, "Unable to create DB {0}", settings.InfluxDB.DatabaseName);
                    return (int) ExitCode.UnableToProcess;
                }
                switch (settings.FileFormat)
                {
                    case FileFormats.Perfmon:
                        result = new PerfmonFile ().ProcessPerfMonLog (settings.InputFileName, client).Result;
                        break;
                    case FileFormats.Generic:
                        if (String.IsNullOrWhiteSpace (settings.InfluxDB.Measurement))
                            throw new ArgumentException ("Generic format needs TableName input");
                        result = new GenericFile ().ProcessGenericFile (settings.InputFileName, client).Result;
                        break;
                }

                stopwatch.Stop ();
                Logger.LogLine (LogLevel.Info, "\n Finished!! Processed {0} points (Success: {1}, Failed:{2}) in {3}", result.PointsFound, result.PointsProcessed, result.PointsFailed, stopwatch.Elapsed.ToString ());

            }

            catch (AggregateException e)
            {
                Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.InnerException.GetType ().Name, e.InnerException.Message, e.InnerException.StackTrace);
            }

            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Error!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
            }
            return (int) result.ExitCode;
        }
示例#6
0
        public async Task<ProcessStatus> ProcessPerfMonLog (string InputFileName, InfluxDBClient client)
        {
            ProcessStatus result = new ProcessStatus ();
            int linesProcessed = 0;
            int failedLines = 0;

            int failedReqCount = 0;
            try
            {

                var failureReasons = new Dictionary<Type, FailureTracker> ();

                Stopwatch stopwatch = new Stopwatch ();
                stopwatch.Start ();


                var firstLine = File.ReadLines (InputFileName).FirstOrDefault ();

                var firstCol = firstLine.Substring (0, firstLine.IndexOf (','));
                if (!firstCol.Contains ("PDH-CSV"))
                    throw new InvalidDataException ("Input file is not a Standard Perfmon csv file");
                var x = Regex.Matches (firstCol, "([-0-9]+)");
                if (x.Count > 0)
                    minOffset = int.Parse (x[3].ToString ());

                //get the column headers
                List<PerfmonCounter> pecrfCounters;
                try
                {
                    pecrfCounters = ParsePerfMonFileHeader (firstLine);
                }
                catch (Exception ex)
                {
                    throw new InvalidDataException ("Unable to parse file headers", ex);
                }



                IInfluxDatabase dbStructure;
                IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup;
                if (settings.PerfmonFile.Filter != Filters.None)
                {
                    var filterColumns = ParsePerfMonFileHeader (settings.PerfmonFile.ColumnsFilter.Columns.ToString (), false);
                    dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
                    perfGroup = FilterPerfmonLogColumns (pecrfCounters, filterColumns, dbStructure as InfluxDatabase).GroupBy (p => p.PerformanceObject);
                }
                else
                {
                    perfGroup = pecrfCounters.GroupBy (p => p.PerformanceObject);
                }

                List<IInfluxDatapoint> points = null, retryQueue = new List<IInfluxDatapoint> ();
                

                if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                {
                    var policies = await client.GetRetentionPoliciesAsync (settings.InfluxDB.DatabaseName);
                    //if duraiton is specified that takes precidence
                    if (settings.InfluxDB.RetentionDuration != 0)
                    {
                        policy = policies.FirstOrDefault (p => p.Duration.TotalMinutes == settings.InfluxDB.RetentionDuration);

                        if (policy == null)
                        {
                            policy = new InfluxRetentionPolicy ()
                            {
                                Name = String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy) ? $"InfluxerRetention_{settings.InfluxDB.RetentionDuration}min" : settings.InfluxDB.RetentionPolicy,
                                DBName = settings.InfluxDB.DatabaseName,
                                Duration = TimeSpan.FromMinutes (settings.InfluxDB.RetentionDuration),
                                IsDefault = false,
                                ReplicaN = 1
                            };
                            if (!await client.CreateRetentionPolicyAsync (policy))
                                throw new InvalidOperationException ("Unable to create retention policy");
                        }
                    }
                    else if (!String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
                    {
                        policy = policies.FirstOrDefault (p => p.Name == settings.InfluxDB.RetentionPolicy);
                        if (policy == null)
                            throw new ArgumentException ("No Retention policy with Name {0} was found, and duration is not specified to create a new one!!", settings.InfluxDB.RetentionPolicy);
                    }
                }
                //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
                foreach (var line in File.ReadLines (InputFileName).Skip (1))
                {
                    try
                    {
                        var linePoints = ProcessPerfmonLogLine (line, perfGroup);
                        linesProcessed++;

                        if (linePoints == null || linePoints.Count == 0)
                            failedLines++;
                        else
                        {
                            result.PointsFound += linePoints.Count;

                            if (points == null)
                                points = linePoints;
                            else
                                points.AddRange (linePoints);

                            if (points.Count >= settings.InfluxDB.PointsInSingleBatch)
                            {
                                bool postresult = false;
                                try
                                {
                                    postresult = await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points);
                                }
                                catch (ServiceUnavailableException)
                                {
                                    postresult = false;
                                }
                                if (postresult)
                                {
                                    failedReqCount = 0;
                                    result.PointsProcessed += points.Count;
                                }
                                else
                                {
                                    //add failed to retry queue
                                    retryQueue.AddRange (points.Where (p => p.Saved != true));
                                    result.PointsProcessed += points.Count (p => p.Saved);
                                    //avoid failing on too many points
                                    if (++failedReqCount > 4)
                                        break;
                                }
                                //a point will be either posted to Influx or in retry queue
                                points = null;
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        failedLines++;
                        var type = e.GetType ();
                        if (!failureReasons.ContainsKey (type))
                            failureReasons.Add (type, new FailureTracker () { ExceptionType = type, Message = e.Message });
                        failureReasons[type].LineNumbers.Add (linesProcessed);
                    }

                    if (failedLines > 0 || retryQueue.Count > 0)
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points     ", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    else
                        Logger.Log (LogLevel.Verbose, "\r{0} Processed:- {1} lines, {2} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound);

                }

                //if we reached here due to repeated failures
                if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || failedReqCount > 3)
                    throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");

                //finally few points may be left out which were not processed (say 10 points left, but we check for 100 points in a batch)
                if (points != null && points.Count > 0)
                {

                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, points))
                    {
                        result.PointsProcessed += points.Count;
                        points.Clear ();
                    }
                    else
                    {
                        failedReqCount++;
                        //add failed to retry queue
                        retryQueue.AddRange (points.Where (p => p.Saved != true));
                        result.PointsProcessed += points.Count (p => p.Saved);
                    }
                }

                if (retryQueue.Count > 0)
                {
                    Logger.LogLine (LogLevel.Info, "\n {0} Retrying {1} failed points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), retryQueue.Count);
                    if (await client.PostPointsAsync (settings.InfluxDB.DatabaseName, retryQueue))
                    {
                        result.PointsProcessed += retryQueue.Count;
                        retryQueue.Clear ();
                    }
                    else if (retryQueue.Count >= settings.InfluxDB.PointsInSingleBatch * 3 || ++failedReqCount > 4)
                        throw new InvalidOperationException ("InfluxDB is not able to accept points!! Please check InfluxDB logs for error details!");
                    else
                    {
                        result.PointsFailed += retryQueue.Count;
                    }
                }

                pecrfCounters.Clear ();
                stopwatch.Stop ();
                if (failedLines > 0 || retryQueue.Count > 0)
                {
                    Logger.Log (LogLevel.Verbose, "\r{0} Done!! Processed:- {1} lines, {2} points, Failed:- {3} lines, {4} points", stopwatch.Elapsed.ToString (@"hh\:mm\:ss"), linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    Logger.LogLine (LogLevel.Error, "Process Started {0}, Input {1}, Processed:- {2} lines, {3} points, Failed:- {4} lines, {5} points", (DateTime.Now - stopwatch.Elapsed), InputFileName, linesProcessed, result.PointsFound, failedLines, retryQueue.Count);
                    foreach (var f in failureReasons.Values)
                        Logger.LogLine (LogLevel.Error, "{0} lines ({1}) failed due to {2} ({3})", f.Count, String.Join (",", f.LineNumbers), f.ExceptionType, f.Message);
                    if (failedLines == linesProcessed || result.PointsFound == retryQueue.Count)
                        result.ExitCode = ExitCode.UnableToProcess;
                    else
                        result.ExitCode = ExitCode.ProcessedWithErrors;
                }
                else
                {
                    result.ExitCode = ExitCode.Success;
                    Logger.LogLine (LogLevel.Info, "\n Done!! Processed:- {0} lines, {1} points", linesProcessed, result.PointsFound);
                }
            }

            catch (Exception e)
            {
                Logger.LogLine (LogLevel.Error, "Failed to process {0}", InputFileName);
                Logger.LogLine (LogLevel.Error, "\r\nError!! {0}:{1} - {2}", e.GetType ().Name, e.Message, e.StackTrace);
                result.ExitCode = ExitCode.UnknownError;
            }
            
            return result;
        }