public void AddLayoutRegexps(IEnumerable <SyntaxAnalysis.NodeRegex> regexps, string layoutId)
{
foreach (var re in regexps)
{
List <CapturedNodeRegex> capturesList;
string capturePrefix;
if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.IsIgnorable) != 0)
{
log.AddMessage(ImportLog.MessageType.RendererIgnored, ImportLog.MessageSeverity.Warn).AddText(
"Renderer").AddCustom(re.AddLinkToSelf).AddText("was ignored");
capturesList = null;
capturePrefix = null;
}
else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime) != 0)
{
capturesList = dateTimeRegexps;
capturePrefix = "time";
}
else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsSeverity) != 0)
{
capturesList = severityRegexps;
capturePrefix = "sev";
}
else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsThread) != 0)
{
capturesList = threadRegexps;
capturePrefix = "thread";
}
else if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.IsAuxiliaryRegexPart) != 0)
{
capturesList = null;
capturePrefix = null;
}
else
{
capturesList = otherRegexps;
capturePrefix = "content";
}
if (headerReBuilder.Length > 0)
{
headerReBuilder.Append(Environment.NewLine);
}
if (capturePrefix != null)
{
string captureName = string.Format("{0}{1}", capturePrefix, capturesList.Count + 1);
capturesList.Add(new CapturedNodeRegex()
{
Regex = re, CaptureName = captureName, LayoutId = layoutId
});
headerReBuilder.AppendFormat("(?<{0}>{1}) # {2}", captureName, re.Regex, EscapeRegexComment(re.NodeDescription) ?? "");
}
else
{
headerReBuilder.AppendFormat("{0} # {1}", re.Regex, EscapeRegexComment(re.NodeDescription) ?? "");
}
}
}
private static void ReportUnknownRenderers(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
{
foreach (var unknown in regexps.Where(re => (re.Flags & SyntaxAnalysis.NodeRegexFlags.IsUnknownRenderer) != 0))
{
log.AddMessage(ImportLog.MessageType.UnknownRenderer, ImportLog.MessageSeverity.Warn)
.AddText("Unknown renderer")
.AddCustom(unknown.AddLinkToSelf)
.AddText("ignored");
}
}
private static void ReportMatchabilityProblems(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
{
Func <int, bool> isSpecificByIdx = i =>
i >= 0 && i < regexps.Count && (regexps[i].Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) == 0;
Action <int> markAsMakingPreviousCapturable = i =>
{
if (i >= 0 && i < regexps.Count)
{
var tmp = regexps[i];
tmp.Flags |= SyntaxAnalysis.NodeRegexFlags.MakesPreviousCapturable;
regexps[i] = tmp;
}
};
for (int i = 0; i < regexps.Count; ++i)
{
var re = regexps[i];
if ((re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentationMask) != 0 &&
(re.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0)
{
bool isPreceededBySpecific = isSpecificByIdx(i - 1);
bool isFollowedBySpecific = isSpecificByIdx(i + 1);
bool isSurroundedBySpecificNodes = isPreceededBySpecific && isFollowedBySpecific;
bool isNotSurroundedBySpecificNodes = !isSurroundedBySpecificNodes;
bool representsFieldThatCanBeNotSpecific =
(re.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentationMask) == SyntaxAnalysis.NodeRegexFlags.RepresentsThread;
bool representsFieldThatMustBeSpecific = !representsFieldThatCanBeNotSpecific;
if (isNotSurroundedBySpecificNodes || representsFieldThatMustBeSpecific)
{
var warn = log.AddMessage(ImportLog.MessageType.RendererIgnored, ImportLog.MessageSeverity.Warn);
warn
.AddText("Renderer")
.AddCustom(re.AddLinkToSelf)
.AddText("can not be matched and as such ignored.");
if (re.WrapperThatMakesRegexNotSpecific != null)
{
var wrap = re.WrapperThatMakesRegexNotSpecific.Value;
warn
.AddText("Renderer is not matchable because of")
.AddCustom(wrap.AddLinkToSelf);
}
re.Flags = re.Flags & ~SyntaxAnalysis.NodeRegexFlags.RepresentationMask;
regexps[i] = re;
}
if (representsFieldThatCanBeNotSpecific && isSurroundedBySpecificNodes)
{
markAsMakingPreviousCapturable(i + 1);
}
}
}
}
static void ReportRendererUsage(
CapturedNodeRegex rendererNodeRe,
string outputFieldDescription,
ImportLog log)
{
var re = rendererNodeRe;
log.AddMessage(ImportLog.MessageType.RendererUsageReport, ImportLog.MessageSeverity.Info)
.AddText("Renderer")
.AddCustom(re.Regex.AddLinkToSelf)
.AddTextFmt("was used to parse {0}", outputFieldDescription)
.SetLayoutId(rendererNodeRe.LayoutId);
}
private static void ValidateFirstHeaderRegex(List <SyntaxAnalysis.NodeRegex> regexps, ImportLog log)
{
var first = regexps.First();
if ((first.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0)
{
log.AddMessage(ImportLog.MessageType.FirstRegexIsNotSpecific, ImportLog.MessageSeverity.Error)
.AddText("LogJoint can not match layouts that start from")
.AddCustom(first.AddLinkToSelf)
.AddText(". Start your layout with a specific renderer like ${longdate}.");
log.FailIfThereIsError();
}
}
public static void GenerateCsvLayoutConfig(
XmlElement root,
CsvParams csvParams,
ImportLog log
)
{
if (csvParams.FalalLoadingError != null)
{
log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddText(csvParams.FalalLoadingError);
log.FailIfThereIsError();
}
var columnsRegexps = csvParams.ColumnLayouts.ToDictionary(column => column.Key, column => ParseLayout(column.Value));
var escapingOptions = GetCsvEscapingOptions(csvParams);
var delimiterRegex = GetCsvDelimiterRegex(csvParams);
var configBuilder = new ConfigBuilder(log, escapingOptions);
configBuilder.HeaderReBuilder.Append("^");
int columnIdx = 0;
foreach (var column in columnsRegexps)
{
using (new ScopedGuard(() => log.StartHandlingLayout(column.Key), () => log.StopHandlingLayout()))
{
if (columnIdx == 0)
{
ValidateFirstHeaderRegex(column.Value, log);
}
ReportUnknownRenderers(column.Value, log);
ReportMatchabilityProblems(column.Value, log);
if (columnIdx > 0)
{
configBuilder.HeaderReBuilder.AppendFormat(
"{0}{1} # CSV separator", Environment.NewLine, delimiterRegex);
}
configBuilder.HeaderReBuilder.Append(escapingOptions.QuoteRegex);
configBuilder.AddLayoutRegexps(column.Value, column.Key);
configBuilder.HeaderReBuilder.Append(escapingOptions.QuoteRegex);
++columnIdx;
}
}
configBuilder.GenerateConfig(root);
}
static void WarnAboutConditionalRenderer(
CapturedNodeRegex rendererNodeRe,
string rendererDescription,
ImportLog log)
{
var re = rendererNodeRe;
Debug.Assert((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0);
var warn = log.AddMessage(ImportLog.MessageType.ImportantFieldIsConditional, ImportLog.MessageSeverity.Warn)
.AddText(rendererDescription)
.AddCustom(re.Regex.AddLinkToSelf)
.AddTextFmt("is not guaranteed to produce output")
.SetLayoutId(rendererNodeRe.LayoutId);
if (re.Regex.WrapperThatMakesRegexConditional != null)
{
warn.AddText("because it is wrapped by conditional renderer")
.AddCustom(re.Regex.WrapperThatMakesRegexConditional.Value.AddLinkToSelf)
.SetLayoutId(rendererNodeRe.LayoutId);
}
}
public static void GenerateJsonLayoutConfig(
XmlElement root,
JsonParams jsonParams,
ImportLog log
)
{
if (jsonParams.FalalLoadingError != null)
{
log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddText(jsonParams.FalalLoadingError);
log.FailIfThereIsError();
}
var configBuilder = new ConfigBuilder(log, new EscapingOptions()
{
EscapingFormat = "JSON_UNESCAPE({0})"
});
configBuilder.HeaderReBuilder.Append("^");
Action <JsonParams.Layout> handleLayout = null;
handleLayout = (layout) =>
{
string spacesRegex = layout.SuppressSpaces ? "" : "\\s";
configBuilder.HeaderReBuilder.AppendFormat("{0}{1}{2} # json layout begin", Environment.NewLine, '{', spacesRegex);
foreach (var attr in layout.Attrs)
{
configBuilder.HeaderReBuilder.AppendFormat("{0}( # begin of optional group for attr '{1}'", Environment.NewLine, attr.Key);
configBuilder.HeaderReBuilder.AppendFormat("{0}(\\,{1})? # comma between attrs", Environment.NewLine, spacesRegex);
configBuilder.HeaderReBuilder.AppendFormat("{0}\"{1}\":{2} # name of attr '{3}'", Environment.NewLine, Regex.Escape(attr.Key), spacesRegex, attr.Key);
bool attributeCanBeMissing = true;
if (attr.Value.SimpleLayout != null)
{
using (new ScopedGuard(() => log.StartHandlingLayout(attr.Value.Id), () => log.StopHandlingLayout()))
{
var regexps = ParseLayout(attr.Value.SimpleLayout);
ReportUnknownRenderers(regexps, log);
ReportMatchabilityProblems(regexps, log);
attributeCanBeMissing = regexps.All(r => (r.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) != 0);
configBuilder.HeaderReBuilder.AppendFormat("{0}\" # value of '{1}' begins", Environment.NewLine, attr.Key);
configBuilder.AddLayoutRegexps(regexps, attr.Value.Id);
configBuilder.HeaderReBuilder.AppendFormat(@"{0}(?<!\\)"" # value of '{1}' ends", Environment.NewLine, attr.Key);
}
}
else
{
if (attr.Value.Encode)
{
log.AddMessage(ImportLog.MessageType.BadLayout, ImportLog.MessageSeverity.Error).AddTextFmt(
"Attribute '{0}' with nested JSON layout is configured to JSON-encode the output (encode=true). Parsing of such layouts is not supported by LogJoint",
attr.Key
);
}
attributeCanBeMissing = !attr.Value.JsonLayout.RenderEmptyObject;
handleLayout(attr.Value.JsonLayout);
}
configBuilder.HeaderReBuilder.AppendFormat("{0}){2} # end of group for attr '{1}'",
Environment.NewLine, attr.Key, attributeCanBeMissing ? "?" : "");
}
if (layout.IncludeAllProperties || layout.IncludeMdc || layout.IncludeMdlc)
{
configBuilder.HeaderReBuilder.AppendFormat("{0}(\\,.+?)? # optional extra attributes", Environment.NewLine);
}
configBuilder.HeaderReBuilder.AppendFormat("{0}{1}{2} # json layout end", Environment.NewLine, spacesRegex, '}');
};
handleLayout(jsonParams.Root);
configBuilder.GenerateConfig(root);
}
private static string GetDateTimeCode(List <CapturedNodeRegex> dateTimeRegexps, ImportLog log)
{
Debug.Assert(dateTimeRegexps.All(re => (re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsNotSpecific) == 0),
"Checked in ValidateRegexpsList()");
StringBuilder dateTimeCode = new StringBuilder();
Func <SyntaxAnalysis.NodeRegexFlags, bool, CapturedNodeRegex> findDateTimeRe = (representationMask, isConditional) =>
dateTimeRegexps.FirstOrDefault(re =>
(re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime) == representationMask &&
((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0) == isConditional);
Func <CapturedNodeRegex, string> getDateTimeExpression = re =>
{
if (re.Regex.DateTimeFormat == SyntaxAnalysis.TicksFakeDateTimeFormat)
{
return(string.Format("TICKS_TO_DATETIME({0})", re.CaptureName));
}
string fmt = re.Regex.DateTimeCulture == null ? "TO_DATETIME({0}, {1})" : "TO_DATETIME({0}, {1}, \"{2}\")";
return(string.Format(fmt, re.CaptureName, StringUtils.GetCSharpStringLiteral(re.Regex.DateTimeFormat), re.Regex.DateTimeCulture));
};
var concreteDateTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime, false);
var concreteDate = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDate, false);
var concreteTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsTime, false);
//var conditionalDateTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDateOrTime, true);
//var conditionalDate = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsDate, true);
//var conditionalTime = findDateTimeRe(SyntaxAnalysis.NodeRegexFlags.RepresentsTime, true);
var used = new List <CapturedNodeRegex>();
if (concreteDateTime != null)
{
dateTimeCode.Append(getDateTimeExpression(concreteDateTime));
used.Add(concreteDateTime);
}
else if (concreteDate != null && concreteTime != null)
{
dateTimeCode.AppendFormat("DATETIME_FROM_DATE_AND_TIMEOFDAY({0}, {1})",
getDateTimeExpression(concreteDate), getDateTimeExpression(concreteTime));
used.Add(concreteDate);
used.Add(concreteTime);
}
else if (concreteDate != null)
{
dateTimeCode.Append(getDateTimeExpression(concreteDate));
log.AddMessage(ImportLog.MessageType.NoTimeParsed, ImportLog.MessageSeverity.Warn).AddText(
"No time renderer found in the layout. Messages timestamp will have 1 day precision.");
used.Add(concreteDate);
}
else if (concreteTime != null)
{
dateTimeCode.AppendFormat("DATETIME_FROM_TIMEOFDAY({0})", getDateTimeExpression(concreteTime));
used.Add(concreteTime);
}
else if (dateTimeRegexps.Count == 0)
{
log.AddMessage(ImportLog.MessageType.NoDateTimeFound, ImportLog.MessageSeverity.Error).AddText(
"Mandatory matchable date/time renderer not found in the layout");
}
else
{
foreach (var re in dateTimeRegexps)
{
if ((re.Regex.Flags & SyntaxAnalysis.NodeRegexFlags.IsConditional) != 0)
{
WarnAboutConditionalRenderer(re, "date/time renderer", log);
}
}
log.AddMessage(ImportLog.MessageType.DateTimeCannotBeParsed, ImportLog.MessageSeverity.Error).AddText(
"Layout string contains date/time renderer(s) but they can not be used to reliably parse mandatory message timestamp");
}
log.FailIfThereIsError();
foreach (var re in used)
{
ReportRendererUsage(re, "message timestamp", log);
}
return(dateTimeCode.ToString());
}