public object TestCategorizer(ScriptEngine engine, object state, string /*!*/ src, SourceLocation initial, params TokenInfo[] /*!*/ expected) { TokenCategorizer categorizer = engine.GetService <TokenCategorizer>(); categorizer.Initialize(state, engine.CreateScriptSourceFromString(src), initial); IEnumerable <TokenInfo> actual = categorizer.ReadTokens(Int32.MaxValue); int i = 0; foreach (TokenInfo info in actual) { Assert(i < expected.Length); if (!info.Equals(expected[i])) { Assert(false); } i++; } Assert(i == expected.Length); TokenInfo t = categorizer.ReadToken(); SourceLocation end = expected[expected.Length - 1].SourceSpan.End; Assert(t.Equals(new TokenInfo(new SourceSpan(end, end), TokenCategory.EndOfStream, TokenTriggers.None))); return(categorizer.CurrentState); }
public bool TryMatchContextPossibility(int index, IReverseTokenizer revTokenizer, out IEnumerable <ContextPossibility> matchingPossibilities) { List <ContextPossibility> retList = new List <ContextPossibility>(); bool isMatch = false; if (_flatMatchingSet.Count > 0) { // start reverse parsing IEnumerator <TokenInfo> enumerator = revTokenizer.GetReversedTokens().Where(x => x.SourceSpan.Start.Index < index).GetEnumerator(); while (true) { if (!enumerator.MoveNext()) { isMatch = false; break; } TokenInfo tokenInfo = enumerator.Current; if (tokenInfo.Equals(default(TokenInfo)) || tokenInfo.Token.Kind == TokenKind.NewLine || tokenInfo.Token.Kind == TokenKind.NLToken || tokenInfo.Token.Kind == TokenKind.Comment) { continue; // linebreak } string info = tokenInfo.ToString(); // look for the token in the matching dictionary List <BackwardTokenSearchItem> matchList; if (_flatMatchingSet.TryGetValue(tokenInfo.Token.Kind, out matchList) || _flatMatchingSet.TryGetValue(tokenInfo.Category, out matchList)) { // need to attempt matching the match list // 1) grab the potential matches with an ordered set and attempt to completely match each set. If one of the sets completely matches, we have a winner foreach (var potentialMatch in matchList.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion && x.TokenSet != null)) { if (AttemptOrderedSetMatch(tokenInfo.SourceSpan.Start.Index, revTokenizer, potentialMatch.TokenSet) && _parentTree.LanguageVersion >= potentialMatch.ParentContext.MinimumLanguageVersion) { retList.Add(potentialMatch.ParentContext); isMatch = true; break; } } if (isMatch) { break; // if we have a match from the for loop above, we're done. } // 2) If we have any single token matches, they win. var singleMatches = matchList.Where(x => (x.SingleToken is TokenKind && (TokenKind)x.SingleToken != TokenKind.EndOfFile) || (x.SingleToken is TokenCategory && (TokenCategory)x.SingleToken != TokenCategory.None)).ToList(); if (singleMatches.Count > 0) { retList.AddRange(singleMatches.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion).Select(x => x.ParentContext)); isMatch = true;// singleMatches.All(x => x.Match); // TODO: the match flag isn't being used....need to figure that out. break; } // At this point, nothing was matched correctly, so we continue } else if (_flatNonMatchingSet.Count > 0 && (!_flatNonMatchingSet.Contains(tokenInfo.Token.Kind) || !_flatNonMatchingSet.Contains(tokenInfo.Category))) { // need to attempt matching the match list // 1) grab the potential matches with an ordered set and attempt to completely match each set. If one of the sets completely matches, we have a winner foreach (var potentialMatch in _flatNonMatchingSet.SelectMany(x => _flatMatchingSet[x]).Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion && x.TokenSet != null)) { if (AttemptOrderedSetMatch(tokenInfo.SourceSpan.Start.Index, revTokenizer, potentialMatch.TokenSet, false) && _parentTree.LanguageVersion >= potentialMatch.ParentContext.MinimumLanguageVersion) { retList.Add(potentialMatch.ParentContext); isMatch = true; break; } } if (isMatch) { break; // if we have a match from the for loop above, we're done. } // 2) If we have any single token matches, they win. var singleMatches = _flatNonMatchingSet.SelectMany(x => _flatMatchingSet[x]).Where(x => (x.SingleToken is TokenKind && (TokenKind)x.SingleToken != TokenKind.EndOfFile) || (x.SingleToken is TokenCategory && (TokenCategory)x.SingleToken != TokenCategory.None)).ToList(); if (singleMatches.Count > 0) { retList.AddRange(singleMatches.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion).Select(x => x.ParentContext)); isMatch = true; break; } // At this point, nothing was matched correctly, so we continue } else { if (Genero4glAst.ValidStatementKeywords.Contains(tokenInfo.Token.Kind)) { isMatch = false; break; } } } } if (!isMatch && _possibilitiesWithNoBackwardSearch.Count > 0) { retList.AddRange(_possibilitiesWithNoBackwardSearch.Where(x => _parentTree.LanguageVersion >= x.MinimumLanguageVersion)); isMatch = true; } matchingPossibilities = retList; return(isMatch); }