예제 #1
0
            public void AddLayoutRegexps(IEnumerable <SyntaxAnalysis.NodeRegex> regexps, string layoutId)
            {
                foreach (var re in regexps)
                {
                    List <CapturedNodeRegex> capturesList;
                    string capturePrefix;

                    if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.IsIgnorable) != 0)
                    {
                        log.AddMessage(ImportLog.MessageType.RendererIgnored, ImportLog.MessageSeverity.Warn).AddText(
                            "Renderer").AddCustom(re.AddLinkToSelf).AddText("was ignored");
                        capturesList  = null;
                        capturePrefix = null;
                    }
                    else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime) != 0)
                    {
                        capturesList  = dateTimeRegexps;
                        capturePrefix = "time";
                    }
                    else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsSeverity) != 0)
                    {
                        capturesList  = severityRegexps;
                        capturePrefix = "sev";
                    }
                    else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsThread) != 0)
                    {
                        capturesList  = threadRegexps;
                        capturePrefix = "thread";
                    }
                    else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.IsAuxiliaryRegexPart) != 0)
                    {
                        capturesList  = null;
                        capturePrefix = null;
                    }
                    else
                    {
                        capturesList  = otherRegexps;
                        capturePrefix = "content";
                    }

                    if (headerReBuilder.Length > 0)
                    {
                        headerReBuilder.Append(Environment.NewLine);
                    }

                    if (capturePrefix != null)
                    {
                        string captureName = string.Format("{0}{1}", capturePrefix, capturesList.Count + 1);
                        capturesList.Add(new CapturedNodeRegex()
                        {
                            Regex = re, CaptureName = captureName, LayoutId = layoutId
                        });
                        headerReBuilder.AppendFormat("(?<{0}>{1}) # {2}", captureName, re.Regex, EscapeRegexComment(re.NodeDescription) ?? "");
                    }
                    else
                    {
                        headerReBuilder.AppendFormat("{0} # {1}", re.Regex, EscapeRegexComment(re.NodeDescription) ?? "");
                    }
                }
            }
예제 #2
0
 private static void ReportUnknownRenderers(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
 {
     foreach (var unknown in regexps.Where(re => (re.Flags & SyntaxAnalysis.NodeRegexFlags.IsUnknownRenderer) != 0))
     {
         log.AddMessage(ImportLog.MessageType.UnknownRenderer, ImportLog.MessageSeverity.Warn)
         .AddText("Unknown renderer")
         .AddCustom(unknown.AddLinkToSelf)
         .AddText("ignored");
     }
 }
예제 #3
0
        private static void ReportMatchabilityProblems(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
        {
            Func <int, bool> isSpecificByIdx = i =>
                                               i >= 0 && i < regexps.Count && (regexps[i].Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) == 0;

            Action <int> markAsMakingPreviousCapturable = i =>
            {
                if (i >= 0 && i < regexps.Count)
                {
                    var tmp = regexps[i];
                    tmp.Flags |= SyntaxAnalysis.NodeRegexFlags.MakesPreviousCapturable;
                    regexps[i] = tmp;
                }
            };

            for (int i = 0; i < regexps.Count; ++i)
            {
                var re = regexps[i];
                if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentationMask) != 0 &&
                    (re.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0)
                {
                    bool isPreceededBySpecific               = isSpecificByIdx(i - 1);
                    bool isFollowedBySpecific                = isSpecificByIdx(i + 1);
                    bool isSurroundedBySpecificNodes         = isPreceededBySpecific && isFollowedBySpecific;
                    bool isNotSurroundedBySpecificNodes      = !isSurroundedBySpecificNodes;
                    bool representsFieldThatCanBeNotSpecific =
                        (re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentationMask) == SyntaxAnalysis.NodeRegexFlags.RepresentsThread;
                    bool representsFieldThatMustBeSpecific = !representsFieldThatCanBeNotSpecific;
                    if (isNotSurroundedBySpecificNodes || representsFieldThatMustBeSpecific)
                    {
                        var warn = log.AddMessage(ImportLog.MessageType.RendererIgnored, ImportLog.MessageSeverity.Warn);
                        warn
                        .AddText("Renderer")
                        .AddCustom(re.AddLinkToSelf)
                        .AddText("can not be matched and as such ignored.");
                        if (re.WrapperThatMakesRegexNotSpecific != null)
                        {
                            var wrap = re.WrapperThatMakesRegexNotSpecific.Value;
                            warn
                            .AddText("Renderer is not matchable because of")
                            .AddCustom(wrap.AddLinkToSelf);
                        }

                        re.Flags   = re.Flags & ~SyntaxAnalysis.NodeRegexFlags.RepresentationMask;
                        regexps[i] = re;
                    }
                    if (representsFieldThatCanBeNotSpecific && isSurroundedBySpecificNodes)
                    {
                        markAsMakingPreviousCapturable(i + 1);
                    }
                }
            }
        }
예제 #4
0
        static void ReportRendererUsage(
            CapturedNodeRegex rendererNodeRe,
            string outputFieldDescription,
            ImportLog log)
        {
            var re = rendererNodeRe;

            log.AddMessage(ImportLog.MessageType.RendererUsageReport, ImportLog.MessageSeverity.Info)
            .AddText("Renderer")
            .AddCustom(re.Regex.AddLinkToSelf)
            .AddTextFmt("was used to parse {0}", outputFieldDescription)
            .SetLayoutId(rendererNodeRe.LayoutId);
        }
예제 #5
0
        private static void ValidateFirstHeaderRegex(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
        {
            var first = regexps.First();

            if ((first.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0)
            {
                log.AddMessage(ImportLog.MessageType.FirstRegexIsNotSpecific, ImportLog.MessageSeverity.Error)
                .AddText("LogJoint can not match layouts that start from")
                .AddCustom(first.AddLinkToSelf)
                .AddText(". Start your layout with a specific renderer like ${longdate}.");
                log.FailIfThereIsError();
            }
        }
예제 #6
0
        public static void GenerateCsvLayoutConfig(
            XmlElement root,
            CsvParams csvParams,
            ImportLog log
            )
        {
            if (csvParams.FalalLoadingError != null)
            {
                log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddText(csvParams.FalalLoadingError);
                log.FailIfThereIsError();
            }

            var columnsRegexps = csvParams.ColumnLayouts.ToDictionary(column => column.Key, column => ParseLayout(column.Value));

            var escapingOptions = GetCsvEscapingOptions(csvParams);
            var delimiterRegex  = GetCsvDelimiterRegex(csvParams);

            var configBuilder = new ConfigBuilder(log, escapingOptions);

            configBuilder.HeaderReBuilder.Append("^");
            int columnIdx = 0;

            foreach (var column in columnsRegexps)
            {
                using (new ScopedGuard(() => log.StartHandlingLayout(column.Key), () => log.StopHandlingLayout()))
                {
                    if (columnIdx == 0)
                    {
                        ValidateFirstHeaderRegex(column.Value, log);
                    }
                    ReportUnknownRenderers(column.Value, log);
                    ReportMatchabilityProblems(column.Value, log);

                    if (columnIdx > 0)
                    {
                        configBuilder.HeaderReBuilder.AppendFormat(
                            "{0}{1} # CSV separator", Environment.NewLine, delimiterRegex);
                    }

                    configBuilder.HeaderReBuilder.Append(escapingOptions.QuoteRegex);

                    configBuilder.AddLayoutRegexps(column.Value, column.Key);

                    configBuilder.HeaderReBuilder.Append(escapingOptions.QuoteRegex);

                    ++columnIdx;
                }
            }

            configBuilder.GenerateConfig(root);
        }
예제 #7
0
        static void WarnAboutConditionalRenderer(
            CapturedNodeRegex rendererNodeRe,
            string rendererDescription,
            ImportLog log)
        {
            var re = rendererNodeRe;

            Debug.Assert((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0);
            var warn = log.AddMessage(ImportLog.MessageType.ImportantFieldIsConditional, ImportLog.MessageSeverity.Warn)
                       .AddText(rendererDescription)
                       .AddCustom(re.Regex.AddLinkToSelf)
                       .AddTextFmt("is not guaranteed to produce output")
                       .SetLayoutId(rendererNodeRe.LayoutId);

            if (re.Regex.WrapperThatMakesRegexConditional != null)
            {
                warn.AddText("because it is wrapped by conditional renderer")
                .AddCustom(re.Regex.WrapperThatMakesRegexConditional.Value.AddLinkToSelf)
                .SetLayoutId(rendererNodeRe.LayoutId);
            }
        }
예제 #8
0
        public static void GenerateJsonLayoutConfig(
            XmlElement root,
            JsonParams jsonParams,
            ImportLog log
            )
        {
            if (jsonParams.FalalLoadingError != null)
            {
                log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddText(jsonParams.FalalLoadingError);
                log.FailIfThereIsError();
            }

            var configBuilder = new ConfigBuilder(log, new EscapingOptions()
            {
                EscapingFormat = "JSON_UNESCAPE({0})"
            });

            configBuilder.HeaderReBuilder.Append("^");

            Action <JsonParams.Layout> handleLayout = null;

            handleLayout = (layout) =>
            {
                string spacesRegex = layout.SuppressSpaces ? "" : "\\s";

                configBuilder.HeaderReBuilder.AppendFormat("{0}{1}{2} # json layout begin", Environment.NewLine, '{', spacesRegex);
                foreach (var attr in layout.Attrs)
                {
                    configBuilder.HeaderReBuilder.AppendFormat("{0}( # begin of optional group for attr '{1}'", Environment.NewLine, attr.Key);
                    configBuilder.HeaderReBuilder.AppendFormat("{0}(\\,{1})? # comma between attrs", Environment.NewLine, spacesRegex);
                    configBuilder.HeaderReBuilder.AppendFormat("{0}\"{1}\":{2} # name of attr '{3}'", Environment.NewLine, Regex.Escape(attr.Key), spacesRegex, attr.Key);
                    bool attributeCanBeMissing = true;
                    if (attr.Value.SimpleLayout != null)
                    {
                        using (new ScopedGuard(() => log.StartHandlingLayout(attr.Value.Id), () => log.StopHandlingLayout()))
                        {
                            var regexps = ParseLayout(attr.Value.SimpleLayout);
                            ReportUnknownRenderers(regexps, log);
                            ReportMatchabilityProblems(regexps, log);

                            attributeCanBeMissing = regexps.All(r => (r.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0);

                            configBuilder.HeaderReBuilder.AppendFormat("{0}\" # value of '{1}' begins", Environment.NewLine, attr.Key);

                            configBuilder.AddLayoutRegexps(regexps, attr.Value.Id);

                            configBuilder.HeaderReBuilder.AppendFormat(@"{0}(?<!\\)"" # value of '{1}' ends", Environment.NewLine, attr.Key);
                        }
                    }
                    else
                    {
                        if (attr.Value.Encode)
                        {
                            log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddTextFmt(
                                "Attribute '{0}' with nested JSON layout is configured to JSON-encode the output (encode=true). Parsing of such layouts is not supported by LogJoint",
                                attr.Key
                                );
                        }
                        attributeCanBeMissing = !attr.Value.JsonLayout.RenderEmptyObject;
                        handleLayout(attr.Value.JsonLayout);
                    }
                    configBuilder.HeaderReBuilder.AppendFormat("{0}){2} # end of group for attr '{1}'",
                                                               Environment.NewLine, attr.Key, attributeCanBeMissing ? "?" : "");
                }
                if (layout.IncludeAllProperties || layout.IncludeMdc || layout.IncludeMdlc)
                {
                    configBuilder.HeaderReBuilder.AppendFormat("{0}(\\,.+?)? # optional extra attributes", Environment.NewLine);
                }
                configBuilder.HeaderReBuilder.AppendFormat("{0}{1}{2} # json layout end", Environment.NewLine, spacesRegex, '}');
            };

            handleLayout(jsonParams.Root);

            configBuilder.GenerateConfig(root);
        }
예제 #9
0
        private static string GetDateTimeCode(List <CapturedNodeRegex> dateTimeRegexps, ImportLog log)
        {
            Debug.Assert(dateTimeRegexps.All(re => (re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) == 0),
                         "Checked in ValidateRegexpsList()");

            StringBuilder dateTimeCode = new StringBuilder();

            Func <SyntaxAnalysis.NodeRegexFlags, bool, CapturedNodeRegex> findDateTimeRe = (representationMask, isConditional) =>
                                                                                           dateTimeRegexps.FirstOrDefault(re =>
                                                                                                                          (re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime) == representationMask &&
                                                                                                                          ((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0) == isConditional);

            Func <CapturedNodeRegex, string> getDateTimeExpression = re =>
            {
                if (re.Regex.DateTimeFormat == SyntaxAnalysis.TicksFakeDateTimeFormat)
                {
                    return(string.Format("TICKS_TO_DATETIME({0})", re.CaptureName));
                }
                string fmt = re.Regex.DateTimeCulture == null ? "TO_DATETIME({0}, {1})" : "TO_DATETIME({0}, {1}, \"{2}\")";
                return(string.Format(fmt, re.CaptureName, StringUtils.GetCSharpStringLiteral(re.Regex.DateTimeFormat), re.Regex.DateTimeCulture));
            };

            var concreteDateTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime, false);
            var concreteDate     = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDate, false);
            var concreteTime     = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsTime, false);
            //var conditionalDateTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime, true);
            //var conditionalDate = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDate, true);
            //var conditionalTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsTime, true);

            var used = new List <CapturedNodeRegex>();

            if (concreteDateTime != null)
            {
                dateTimeCode.Append(getDateTimeExpression(concreteDateTime));
                used.Add(concreteDateTime);
            }
            else if (concreteDate != null && concreteTime != null)
            {
                dateTimeCode.AppendFormat("DATETIME_FROM_DATE_AND_TIMEOFDAY({0}, {1})",
                                          getDateTimeExpression(concreteDate), getDateTimeExpression(concreteTime));
                used.Add(concreteDate);
                used.Add(concreteTime);
            }
            else if (concreteDate != null)
            {
                dateTimeCode.Append(getDateTimeExpression(concreteDate));
                log.AddMessage(ImportLog.MessageType.NoTimeParsed, ImportLog.MessageSeverity.Warn).AddText(
                    "No time renderer found in the layout. Messages timestamp will have 1 day precision.");
                used.Add(concreteDate);
            }
            else if (concreteTime != null)
            {
                dateTimeCode.AppendFormat("DATETIME_FROM_TIMEOFDAY({0})", getDateTimeExpression(concreteTime));
                used.Add(concreteTime);
            }
            else if (dateTimeRegexps.Count == 0)
            {
                log.AddMessage(ImportLog.MessageType.NoDateTimeFound, ImportLog.MessageSeverity.Error).AddText(
                    "Mandatory matchable date/time renderer not found in the layout");
            }
            else
            {
                foreach (var re in dateTimeRegexps)
                {
                    if ((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0)
                    {
                        WarnAboutConditionalRenderer(re, "date/time renderer", log);
                    }
                }
                log.AddMessage(ImportLog.MessageType.DateTimeCannotBeParsed, ImportLog.MessageSeverity.Error).AddText(
                    "Layout string contains date/time renderer(s) but they can not be used to reliably parse mandatory message timestamp");
            }
            log.FailIfThereIsError();

            foreach (var re in used)
            {
                ReportRendererUsage(re, "message timestamp", log);
            }

            return(dateTimeCode.ToString());
        }