public object TestCategorizer(ScriptEngine engine, object state, string /*!*/ src, SourceLocation initial, params TokenInfo[] /*!*/ expected)
{
TokenCategorizer categorizer = engine.GetService <TokenCategorizer>();
categorizer.Initialize(state, engine.CreateScriptSourceFromString(src), initial);
IEnumerable <TokenInfo> actual = categorizer.ReadTokens(Int32.MaxValue);
int i = 0;
foreach (TokenInfo info in actual)
{
Assert(i < expected.Length);
if (!info.Equals(expected[i]))
{
Assert(false);
}
i++;
}
Assert(i == expected.Length);
TokenInfo t = categorizer.ReadToken();
SourceLocation end = expected[expected.Length - 1].SourceSpan.End;
Assert(t.Equals(new TokenInfo(new SourceSpan(end, end), TokenCategory.EndOfStream, TokenTriggers.None)));
return(categorizer.CurrentState);
}
public bool TryMatchContextPossibility(int index, IReverseTokenizer revTokenizer, out IEnumerable <ContextPossibility> matchingPossibilities)
{
List <ContextPossibility> retList = new List <ContextPossibility>();
bool isMatch = false;
if (_flatMatchingSet.Count > 0)
{
// start reverse parsing
IEnumerator <TokenInfo> enumerator = revTokenizer.GetReversedTokens().Where(x => x.SourceSpan.Start.Index < index).GetEnumerator();
while (true)
{
if (!enumerator.MoveNext())
{
isMatch = false;
break;
}
TokenInfo tokenInfo = enumerator.Current;
if (tokenInfo.Equals(default(TokenInfo)) || tokenInfo.Token.Kind == TokenKind.NewLine || tokenInfo.Token.Kind == TokenKind.NLToken || tokenInfo.Token.Kind == TokenKind.Comment)
{
continue; // linebreak
}
string info = tokenInfo.ToString();
// look for the token in the matching dictionary
List <BackwardTokenSearchItem> matchList;
if (_flatMatchingSet.TryGetValue(tokenInfo.Token.Kind, out matchList) ||
_flatMatchingSet.TryGetValue(tokenInfo.Category, out matchList))
{
// need to attempt matching the match list
// 1) grab the potential matches with an ordered set and attempt to completely match each set. If one of the sets completely matches, we have a winner
foreach (var potentialMatch in matchList.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion && x.TokenSet != null))
{
if (AttemptOrderedSetMatch(tokenInfo.SourceSpan.Start.Index, revTokenizer, potentialMatch.TokenSet) &&
_parentTree.LanguageVersion >= potentialMatch.ParentContext.MinimumLanguageVersion)
{
retList.Add(potentialMatch.ParentContext);
isMatch = true;
break;
}
}
if (isMatch)
{
break; // if we have a match from the for loop above, we're done.
}
// 2) If we have any single token matches, they win.
var singleMatches = matchList.Where(x =>
(x.SingleToken is TokenKind && (TokenKind)x.SingleToken != TokenKind.EndOfFile) ||
(x.SingleToken is TokenCategory && (TokenCategory)x.SingleToken != TokenCategory.None)).ToList();
if (singleMatches.Count > 0)
{
retList.AddRange(singleMatches.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion).Select(x => x.ParentContext));
isMatch = true;// singleMatches.All(x => x.Match); // TODO: the match flag isn't being used....need to figure that out.
break;
}
// At this point, nothing was matched correctly, so we continue
}
else if (_flatNonMatchingSet.Count > 0 &&
(!_flatNonMatchingSet.Contains(tokenInfo.Token.Kind) ||
!_flatNonMatchingSet.Contains(tokenInfo.Category)))
{
// need to attempt matching the match list
// 1) grab the potential matches with an ordered set and attempt to completely match each set. If one of the sets completely matches, we have a winner
foreach (var potentialMatch in _flatNonMatchingSet.SelectMany(x => _flatMatchingSet[x]).Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion && x.TokenSet != null))
{
if (AttemptOrderedSetMatch(tokenInfo.SourceSpan.Start.Index, revTokenizer, potentialMatch.TokenSet, false) &&
_parentTree.LanguageVersion >= potentialMatch.ParentContext.MinimumLanguageVersion)
{
retList.Add(potentialMatch.ParentContext);
isMatch = true;
break;
}
}
if (isMatch)
{
break; // if we have a match from the for loop above, we're done.
}
// 2) If we have any single token matches, they win.
var singleMatches = _flatNonMatchingSet.SelectMany(x => _flatMatchingSet[x]).Where(x =>
(x.SingleToken is TokenKind && (TokenKind)x.SingleToken != TokenKind.EndOfFile) ||
(x.SingleToken is TokenCategory && (TokenCategory)x.SingleToken != TokenCategory.None)).ToList();
if (singleMatches.Count > 0)
{
retList.AddRange(singleMatches.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion).Select(x => x.ParentContext));
isMatch = true;
break;
}
// At this point, nothing was matched correctly, so we continue
}
else
{
if (Genero4glAst.ValidStatementKeywords.Contains(tokenInfo.Token.Kind))
{
isMatch = false;
break;
}
}
}
}
if (!isMatch && _possibilitiesWithNoBackwardSearch.Count > 0)
{
retList.AddRange(_possibilitiesWithNoBackwardSearch.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion));
isMatch = true;
}
matchingPossibilities = retList;
return(isMatch);
}