private TokenizationResult GetSentenceTokens(string sentence)
{
this.EntityLexer.SetInputStream(CharStreams.fromTextReader(new StringReader(sentence)));
this.EntityLexer.RemoveErrorListeners();
TokenizationResult tokenizationResult = new TokenizationResult();
this.EntityLexer.AddErrorListener(new ErrorLexerListener(tokenizationResult, sentence));
tokenizationResult.Tokens = this.EntityLexer.GetAllTokens();
return(tokenizationResult);
}
public ErrorLexerListener(TokenizationResult tokenizationResult, string sentence)
{
this.TokenizationResult = tokenizationResult;
this.Sentence = sentence;
}
private IEnumerable <string> GetSuggestionsFromParser(ATNState parserState, int currentTokenListIndex, TokenizationResult sentenceTokens)
{
if (this.HasVisitedParserState(parserState, currentTokenListIndex))
{
return(new List <string>());
}
int previousTokenListIndex = this.SetParserStateLastVisited(parserState, currentTokenListIndex);
if (!HasMoreTokens(currentTokenListIndex, sentenceTokens))
{
// At this moment, the state in which the sentence ends has been reached.
return(SuggestNextTokensForParserState(parserState, sentenceTokens));
}
foreach (Transition transition in parserState.TransitionsArray)
{
switch (transition)
{
case Transition _ when transition.IsEpsilon:
HandleEpsilonTransition(transition, currentTokenListIndex, sentenceTokens);
break;
case AtomTransition atomTransition:
HandleAtomicTransition(atomTransition, currentTokenListIndex, sentenceTokens);
break;
case SetTransition setTransition:
HandleSetTransition(setTransition, currentTokenListIndex, sentenceTokens);
break;
default:
return(null);
}
}
this.SetParserStateLastVisited(parserState, previousTokenListIndex);
return(new List <string>());
}
private IEnumerable <string> GetSuggestionsFromParser(TokenizationResult sentenceTokens)
{
ATNState initialState = this.EntityParser.Atn.states.FirstOrDefault();
return(this.GetSuggestionsFromParser(initialState, 0, sentenceTokens));
}
private IEnumerable <string> SuggestFromTransitionLabels(HashSet <int> nextParserStateTransitionLabels, TokenizationResult sentenceTokens)
{
List <string> suggestions = new List <string>();
foreach (int nextParserStateTransitionLabel in nextParserStateTransitionLabels)
{
// Count from 0 not from 1.
int nextTokenRuleNumber = nextParserStateTransitionLabel - 1;
ATNState lexerState = this.EntityLexer.Atn.ruleToStartState[nextTokenRuleNumber];
SuggestFromTransitionLabels(lexerState, suggestions, tokensSoFar: string.Empty, remainingText: sentenceTokens.UntokenizedText);
}
return(suggestions);
}
public IEnumerable <string> GetSuggestions(string sentence)
{
TokenizationResult sentenceTokens = this.GetSentenceTokens(sentence);
return(this.GetSuggestionsFromParser(sentenceTokens));
}
private void HandleSetTransition(SetTransition setTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
{
IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);
foreach (int transitionTokenType in setTransition.Label.ToList())
{
if (transitionTokenType == nextToken.TokenIndex)
{
GetSuggestionsFromParser(setTransition.target, currentTokenListIndex + 1, sentenceTokens);
}
}
}
private void HandleAtomicTransition(AtomTransition atomTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
{
IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);
if (atomTransition.Label == sentenceTokens.Tokens.ElementAt(currentTokenListIndex).TokenSource)
{
GetSuggestionsFromParser(atomTransition.target, currentTokenListIndex + 1, sentenceTokens);
}
}
private void HandleEpsilonTransition(Transition transition, int currentTokenListIndex, TokenizationResult sentenceTokens)
{
GetSuggestionsFromParser(transition.target, currentTokenListIndex, sentenceTokens);
}
private static bool HasMoreTokens(int currentTokenListIndex, TokenizationResult sentenceTokens)
{
// Epsilon transitions don't consume a token, so don't move the index
return(currentTokenListIndex > sentenceTokens.Tokens.Count());
}
private IEnumerable <string> SuggestNextTokensForParserState(ATNState reachedParserState, TokenizationResult sentenceTokens)
{
HashSet <int> nextParserStateTransitionLabels = new HashSet <int>();
FillParserTranstionLabels(reachedParserState, nextParserStateTransitionLabels, visitedTransitions: new HashSet <Transition>());
return(SuggestFromTransitionLabels(nextParserStateTransitionLabels, sentenceTokens));
}
