Exemplo n.º 1
0
        private TokenizationResult GetSentenceTokens(string sentence)
        {
            this.EntityLexer.SetInputStream(CharStreams.fromTextReader(new StringReader(sentence)));

            this.EntityLexer.RemoveErrorListeners();

            TokenizationResult tokenizationResult = new TokenizationResult();

            this.EntityLexer.AddErrorListener(new ErrorLexerListener(tokenizationResult, sentence));

            tokenizationResult.Tokens = this.EntityLexer.GetAllTokens();

            return(tokenizationResult);
        }
Exemplo n.º 2
0
 public ErrorLexerListener(TokenizationResult tokenizationResult, string sentence)
 {
     this.TokenizationResult = tokenizationResult;
     this.Sentence           = sentence;
 }
Exemplo n.º 3
0
        private IEnumerable <string> GetSuggestionsFromParser(ATNState parserState, int currentTokenListIndex, TokenizationResult sentenceTokens)
        {
            if (this.HasVisitedParserState(parserState, currentTokenListIndex))
            {
                return(new List <string>());
            }

            int previousTokenListIndex = this.SetParserStateLastVisited(parserState, currentTokenListIndex);

            if (!HasMoreTokens(currentTokenListIndex, sentenceTokens))
            {
                // At this moment, the state in which the sentence ends has been reached.
                return(SuggestNextTokensForParserState(parserState, sentenceTokens));
            }

            foreach (Transition transition in parserState.TransitionsArray)
            {
                switch (transition)
                {
                case Transition _ when transition.IsEpsilon:
                    HandleEpsilonTransition(transition, currentTokenListIndex, sentenceTokens);

                    break;

                case AtomTransition atomTransition:
                    HandleAtomicTransition(atomTransition, currentTokenListIndex, sentenceTokens);

                    break;

                case SetTransition setTransition:
                    HandleSetTransition(setTransition, currentTokenListIndex, sentenceTokens);

                    break;

                default:
                    return(null);
                }
            }

            this.SetParserStateLastVisited(parserState, previousTokenListIndex);

            return(new List <string>());
        }
Exemplo n.º 4
0
        private IEnumerable <string> GetSuggestionsFromParser(TokenizationResult sentenceTokens)
        {
            ATNState initialState = this.EntityParser.Atn.states.FirstOrDefault();

            return(this.GetSuggestionsFromParser(initialState, 0, sentenceTokens));
        }
Exemplo n.º 5
0
        private IEnumerable <string> SuggestFromTransitionLabels(HashSet <int> nextParserStateTransitionLabels, TokenizationResult sentenceTokens)
        {
            List <string> suggestions = new List <string>();

            foreach (int nextParserStateTransitionLabel in nextParserStateTransitionLabels)
            {
                // Count from 0 not from 1.
                int      nextTokenRuleNumber = nextParserStateTransitionLabel - 1;
                ATNState lexerState          = this.EntityLexer.Atn.ruleToStartState[nextTokenRuleNumber];

                SuggestFromTransitionLabels(lexerState, suggestions, tokensSoFar: string.Empty, remainingText: sentenceTokens.UntokenizedText);
            }

            return(suggestions);
        }
Exemplo n.º 6
0
        public IEnumerable <string> GetSuggestions(string sentence)
        {
            TokenizationResult sentenceTokens = this.GetSentenceTokens(sentence);

            return(this.GetSuggestionsFromParser(sentenceTokens));
        }
Exemplo n.º 7
0
        private void HandleSetTransition(SetTransition setTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
        {
            IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);

            foreach (int transitionTokenType in setTransition.Label.ToList())
            {
                if (transitionTokenType == nextToken.TokenIndex)
                {
                    GetSuggestionsFromParser(setTransition.target, currentTokenListIndex + 1, sentenceTokens);
                }
            }
        }
Exemplo n.º 8
0
        private void HandleAtomicTransition(AtomTransition atomTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
        {
            IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);

            if (atomTransition.Label == sentenceTokens.Tokens.ElementAt(currentTokenListIndex).TokenSource)
            {
                GetSuggestionsFromParser(atomTransition.target, currentTokenListIndex + 1, sentenceTokens);
            }
        }
Exemplo n.º 9
0
 private void HandleEpsilonTransition(Transition transition, int currentTokenListIndex, TokenizationResult sentenceTokens)
 {
     GetSuggestionsFromParser(transition.target, currentTokenListIndex, sentenceTokens);
 }
Exemplo n.º 10
0
 private static bool HasMoreTokens(int currentTokenListIndex, TokenizationResult sentenceTokens)
 {
     // Epsilon transitions don't consume a token, so don't move the index
     return(currentTokenListIndex > sentenceTokens.Tokens.Count());
 }
Exemplo n.º 11
0
        private IEnumerable <string> SuggestNextTokensForParserState(ATNState reachedParserState, TokenizationResult sentenceTokens)
        {
            HashSet <int> nextParserStateTransitionLabels = new HashSet <int>();

            FillParserTranstionLabels(reachedParserState, nextParserStateTransitionLabels, visitedTransitions: new HashSet <Transition>());

            return(SuggestFromTransitionLabels(nextParserStateTransitionLabels, sentenceTokens));
        }