public MessagesSplitter(ITextAccess textAccess, IRegex messageHeaderRe, MessagesSplitterFlags flags = MessagesSplitterFlags.Default)
: base()
{
if (textAccess == null)
{
throw new ArgumentNullException(nameof(textAccess));
}
if (messageHeaderRe == null)
{
throw new ArgumentNullException(nameof(messageHeaderRe));
}
if ((messageHeaderRe.Options & ReOptions.RightToLeft) != 0)
{
throw new ArgumentException("Header regular expression must not be RightToLeft");
}
if (textAccess.MaximumSequentialAdvancesAllowed < 3)
{
throw new ArgumentException("ITextAccess implementation must allow 3 or more sequential advances", "textAccess");
}
this.textAccess = textAccess;
this.forwardModeRe = messageHeaderRe;
this.bufferLengthThreshold = 0;
if ((flags & MessagesSplitterFlags.PreventBufferUnderflow) != 0)
{
this.bufferLengthThreshold = textAccess.AverageBufferLength / 4;
}
}
private void AnalyzeCommand(IRegex engine)
{
var disabledSkimmers = new HashSet <string>();
var testLogger = new TestLogger();
// Acquire skimmers for searchers
ISet <Skimmer <AnalyzeContext> > skimmers =
PatternMatcher.AnalyzeCommand.CreateSkimmersFromDefinitionsFiles(
fileSystemMock.Object,
new string[] { searchDefinitionsPath },
engine);
string scanTargetFileName = Path.Combine(@"C:\", Guid.NewGuid().ToString() + ".test");
FlexString fileContents = "bar foo foo";
var context = new AnalyzeContext()
{
TargetUri = new Uri(scanTargetFileName, UriKind.RelativeOrAbsolute),
FileContents = fileContents,
Logger = testLogger
};
IEnumerable <Skimmer <AnalyzeContext> > applicableSkimmers = PatternMatcher.AnalyzeCommand.DetermineApplicabilityForTargetHelper(context, skimmers, disabledSkimmers);
PatternMatcher.AnalyzeCommand.AnalyzeTargetHelper(context, applicableSkimmers, disabledSkimmers);
}
internal Regex(IRegex regex, Func <StringView, int, T> func, bool greedy, Func <Exception> error)
{
this.regex = regex;
this.func = func;
this.greedy = greedy;
this.error = error;
}
public BaseStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, TextStreamPositioningParams textStreamPositioningParams)
{
this.media = media;
this.encoding = encoding;
this.headerRe = headerRe;
this.textStreamPositioningParams = textStreamPositioningParams;
}
public MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe,
MessagesSplitterFlags splitterFlags, TextStreamPositioningParams textStreamPositioningParams,
string parentLoggingPrefix, ITraceSourceFactory traceSourceFactory)
: this(media, encoding, headerRe, splitterFlags, false, textStreamPositioningParams, parentLoggingPrefix, traceSourceFactory)
{
BytesToParsePerThread = GetBytesToParsePerThread(textStreamPositioningParams);
}
static IMessage MakeMessageInternal(
TextMessageCapture capture,
IRegex headRe,
IRegex bodyRe,
ref IMatch bodyMatch,
IFieldsProcessor fieldsProcessor,
MakeMessageFlags makeMessageFlags,
DateTime sourceTime,
ITimeOffsets timeOffsets,
MessagesBuilderCallback threadLocalCallbackImpl
)
{
if (bodyRe != null)
{
if (!bodyRe.Match(capture.BodyBuffer, capture.BodyIndex, capture.BodyLength, ref bodyMatch))
{
return(null);
}
}
int idx = 0;
Group[] groups;
fieldsProcessor.Reset();
fieldsProcessor.SetSourceTime(sourceTime);
fieldsProcessor.SetPosition(capture.BeginPosition);
fieldsProcessor.SetTimeOffsets(timeOffsets);
groups = capture.HeaderMatch.Groups;
for (int i = 1; i < groups.Length; ++i)
{
var g = groups[i];
fieldsProcessor.SetInputField(idx++, new StringSlice(capture.HeaderBuffer, g.Index, g.Length));
}
if (bodyRe != null)
{
groups = bodyMatch.Groups;
for (int i = 1; i < groups.Length; ++i)
{
var g = groups[i];
fieldsProcessor.SetInputField(idx++, new StringSlice(capture.BodyBuffer, g.Index, g.Length));
}
}
else
{
fieldsProcessor.SetInputField(idx++, new StringSlice(capture.BodyBuffer, capture.BodyIndex, capture.BodyLength));
}
threadLocalCallbackImpl.SetCurrentPosition(capture.BeginPosition, capture.EndPosition);
IMessage ret;
ret = fieldsProcessor.MakeMessage(threadLocalCallbackImpl, makeMessageFlags);
ret.SetRawText(StringSlice.Concat(capture.MessageHeaderSlice, capture.MessageBodySlice).Trim());
return(ret);
}
static AwsCredentialsValidator()
{
RegexEngine = RE2Regex.Instance;
Instance = new AwsCredentialsValidator();
RegexEngine.IsMatch(string.Empty, AwsUserExpression);
}
public SingleThreadedStrategyImpl(MessagesReader reader) :
base(reader.LogMedia, reader.StreamEncoding, CloneRegex(reader.formatInfo.HeadRe).Regex,
reader.formatInfo.HeadRe.GetHeaderReSplitterFlags(), reader.formatInfo.TextStreamPositioningParams)
{
this.reader = reader;
this.callback = reader.CreateMessageBuilderCallback();
this.bodyRegex = reader.formatInfo.BodyRe.Regex;
}
public static bool IsMatch(this IRegex c, string pattern, RegexOptions opts)
{
if (c.GetValue().IsNullOrEmpty())
{
return(false);
}
return(Regex.IsMatch(c.GetValue(), pattern, opts));
}
public GeneralMatch(IRegex regex, TextPointer startPoint, TextPointer endPoint, int startOffset, int endOffset)
{
Regex = regex;
StartTextPointer = startPoint;
EndTextPointer = endPoint;
StartOffset = startOffset;
EndOffset = endOffset;
}
static GitHubPatValidator()
{
Instance = new GitHubPatValidator();
RegexEngine = RE2Regex.Instance;
// We perform this work in order to force caching of these
// expressions (an operation which otherwise can cause
// threading problems).
RegexEngine.Match(string.Empty, PatExpression);
}
static SqlConnectionStringValidator()
{
Instance = new SqlConnectionStringValidator();
RegexEngine = RE2Regex.Instance;
// We perform this work in order to force caching of these
// expressions (an operation which otherwise can cause
// threading problems).
RegexEngine.Match(string.Empty, ClientIPExpression);
}
public static IRegex Parse(string regex)
{
var inputStream = new InputStream(regex);
inputStream.Eat(_REGEX_OPENING);
IRegex output = ParseMultiPartRegex(inputStream);
inputStream.Eat(_REGEX_ENDING);
return(output);
}
private static T[] DoArray <T>(Pair previous, IRegex <T> regex, IList <T> sequence, int count) where T : IComparable <T>
{
int length = previous.End - previous.Begin + regex.GetSubsequence(count - 1).Count;
T[] ans = new T[length];
for (int j = 0; j < length; j++)
{
ans[j] = sequence[j + previous.Begin];
}
return(ans);
}
public void ShouldReturnEmptyRegexForEmptyButValidRegexInput()
{
// arrange
string input = "^$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(string.Empty, result.GetLongestNonCyclicWord());
}
public static IRegex CloneRegex(IRegex re, ReOptions optionsToAdd = ReOptions.None)
{
if (re != null)
{
return(re.Factory.Create(re.Pattern, re.Options | optionsToAdd));
}
else
{
return(null);
}
}
public static string Replace(this IRegex c, string parent, string replaceMent, int count, int startAt, RegexOptions opts)
{
var reg = parent.As <IRegex>().ToRegex(opts);
if (count <= 0)
{
return(reg.Replace(c.GetValue(), replaceMent));
}
return(startAt >= 0
? reg.Replace(c.GetValue(), replaceMent, count, startAt)
: reg.Replace(c.GetValue(), replaceMent, count));
}
public InstancedClass(IRegex regex, Func <StringView, int, T> func, bool greedy, Func <Exception> error, Parser <ParseResult> parser, int thisIndex) : base(parser, thisIndex)
{
this.regex = regex;
this.func = func;
this.greedy = greedy;
this.error = error;
this.match = this.regex.FirstMatch;
if (this.greedy)
{
this.match = this.regex.LastMatch;
}
}
public void ShouldReturnCorrectNumberOfNoncyclicWordsForComplexBranches()
{
// arrange
const int expected = 6;
string input = $"^WSSEESWWWNW(S|NENNEEEENN(ESSSSW(NWSW|SSEN)|WSWWN(E|WWS(E|SS))))$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expected, result.GetNonCyclicWords().Count());
}
public void ShouldReturnCorrectNumberOfNoncyclicWordsForOneBranch()
{
// arrange
const int expectedNumberOfWords = 2;
string input = $"^SN(E|TRE)$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expectedNumberOfWords, result.GetNonCyclicWords().Count());
}
public void ShouldReturnValueForOneBranch()
{
// arrange
const string expected = "SNTRE";
string input = $"^SN(E|TRE)$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expected, result.GetLongestNonCyclicWord());
}
public void ShouldReturnResultForThreeCharacterInput()
{
// arrange
const string expecteResult = "SNW";
string input = $"^{expecteResult}$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expecteResult, result.GetLongestNonCyclicWord());
}
public void ShouldReturnCorrectNumberOfNoncyclicWordsForComplexBranchesWithOptional()
{
// arrange
const int expected = 5;
string input = $"^ESSWWN(E|NNENN(EESS(WNSE|)SSS|WWWSSSSE(SW|NNNE)))$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expected, result.GetNonCyclicWords().Count());
}
public void ShouldReturnValueForComplexBranchesWithOptional()
{
// arrange
const string expected = "ESSWWNNNENNWWWSSSSENNNE";
string input = $"^ESSWWN(E|NNENN(EESS(WNSE|)SSS|WWWSSSSE(SW|NNNE)))$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expected, result.GetLongestNonCyclicWord());
}
public void ShouldReturnValueForComplexBranches()
{
// arrange
const string expected = "WSSEESWWWNWNENNEEEENNESSSSWNWSW";
string input = $"^WSSEESWWWNW(S|NENNEEEENN(ESSSSW(NWSW|SSEN)|WSWWN(E|WWS(E|SS))))$";
// act
IRegex result = Parser.Parse(input);
// assert
Assert.AreEqual(expected, result.GetLongestNonCyclicWord());
}
public SearchSkimmer(IRegex engine, ValidatorsCache validators, FileRegionsCache fileRegionsCache, SearchDefinition definition, IFileSystem fileSystem = null)
: this(
engine,
validators,
fileRegionsCache,
definition.Id,
definition.Name,
definition.Description,
definition.MatchExpressions,
fileSystem)
{
}
public static IEnumerable <string> Matches(this IRegex c, string parent, string groupName, RegexOptions opts)
{
var list = new List <string>();
if (c.GetValue().IsNullOrEmpty())
{
return(list);
}
var ms = Regex.Matches(c.GetValue(), parent, opts);
list.AddRange(from Match m in ms select m.Groups[groupName].Value);
return(list);
}
public SingleThreadedStrategyImpl(MessagesReader reader) : base(
reader.LogMedia,
reader.StreamEncoding,
CloneRegex(reader.fmtInfo.HeadRe, reader.IsQuickFormatDetectionMode ? ReOptions.Timeboxed : ReOptions.None).Regex,
reader.fmtInfo.HeadRe.GetHeaderReSplitterFlags(),
reader.fmtInfo.TextStreamPositioningParams
)
{
this.reader = reader;
this.fieldsProcessor = reader.CreateNewFieldsProcessor();
this.callback = reader.CreateMessageBuilderCallback();
this.headerRegex = headerRe;
this.bodyRegex = CloneRegex(reader.fmtInfo.BodyRe).Regex;
}
public SearchSkimmer(
IRegex engine,
ValidatorsCache validators,
FileRegionsCache fileRegionsCache,
string id,
string name,
string description,
IList <MatchExpression> matchExpressions,
IFileSystem fileSystem = null)
{
_id = id;
_name = name;
_engine = engine;
_validators = validators;
_fileRegionsCache = fileRegionsCache;
_fullDescription = new MultiformatMessageString {
Text = description
};
_fileSystem = fileSystem ?? FileSystem.Instance;
_messageStrings = new Dictionary <string, MultiformatMessageString>
{
{ nameof(SdkResources.NotApplicable_InvalidMetadata), new MultiformatMessageString()
{
Text = SdkResources.NotApplicable_InvalidMetadata,
} },
};
foreach (MatchExpression matchExpression in matchExpressions)
{
string matchExpressionMessage = matchExpression.Message;
matchExpression.ArgumentNameToIndexMap = GenerateIndicesForNamedArguments(ref matchExpressionMessage);
string messageId = matchExpression.SubId ?? "Default";
if (!_messageStrings.TryGetValue(messageId, out MultiformatMessageString mfString))
{
_messageStrings[messageId] = new MultiformatMessageString
{
Text = matchExpressionMessage,
};
}
}
_matchExpressions = matchExpressions;
}
internal MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, MessagesSplitterFlags splitterFlags, bool useMockThreading, TextStreamPositioningParams textStreamPositioningParams)
: base(media, encoding, headerRe, textStreamPositioningParams)
{
this.streamDataPool = new ThreadSafeObjectPool <Byte[]>(pool =>
{
var ret = new Byte[BytesToParsePerThread];
tracer.Info("Allocating new piece of stream data: {0}", ret.GetHashCode());
return(ret);
});
this.outputBuffersPool = new ThreadSafeObjectPool <List <PostprocessedMessage> >(pool =>
{
var ret = new List <PostprocessedMessage>(1024 * 8);
tracer.Info("Allocating new output buffer: {0}", ret.GetHashCode());
return(ret);
});
this.useMockThreading = useMockThreading;
this.splitterFlags = splitterFlags;
}
