Beispiel #1
0
        public void Reduce(IProduction production, ReduceVisitor reducer, Lexeme lookahead)
        {
#if HISTORY
            var from = m_topStack.StateIndex;
#endif

            if (production == null)
            {
                //Accept
                Debug.Assert(m_topStack.PrevNode.StateIndex == 0);

                //TODO: accepted
                IsAccepted = true;
                return;
            }

            if (production.AggregatesAmbiguities)
            {
                AmbiguityAggregator = (production as ProductionBase).CreateAggregator();
            }

            var reduceResult = production.Accept(reducer, m_topStack);

            m_topStack = reduceResult.NewTopStack;
            var reduceError = reduceResult.ReduceError;

            if (reduceError != null)
            {
                IncreaseErrorRecoverLevel();

                if (reduceError.ErrorPosition == null)
                {
                    reduceError.ErrorPosition = lookahead.Value.Span;
                }

                AddError(reduceError);
            }

#if HISTORY
            var to = m_topStack.StateIndex;
            History.Add(String.Format("R{0}:{1}", from, to));
#endif
        }
Beispiel #2
0
        private TokenDecl TokenDeclaration()
        {
            var    names = new List <Lexeme>();
            Lexeme n     = lexeme();

            names.Add(n);
            while (LookAhead(EngageToken.COMMA))
            {
                match(EngageToken.COMMA);
                n = lexeme();
                names.Add(n);
            }

            match(EngageToken.IS_TYPE);

            return(new TokenDecl {
                Names = names, Type = consumeText(EngageToken.ID)
            });
        }
Beispiel #3
0
        public void OpenInLexicon(Lexeme lexeme)
        {
            string guid = null, toolName;

            using (m_activationContext.Activate())
            {
                if (lexeme.Type == LexemeType.Word)
                {
                    toolName = "Analyses";
                    IWfiWordform wf;
                    if (TryGetWordform(lexeme.LexicalForm, out wf))
                    {
                        guid = wf.Guid.ToString();
                    }
                }
                else
                {
                    toolName = "lexiconEdit";
                    var       entryLexeme = (FdoLexEntryLexeme)lexeme;
                    ILexEntry entry;
                    if (TryGetEntry(entryLexeme.Key, out entry))
                    {
                        guid = entry.Guid.ToString();
                    }
                }
            }
            if (guid != null)
            {
                string url = string.Format("silfw://localhost/link?app=flex&database={0}&tool={1}&guid={2}",
                                           HttpUtility.UrlEncode(m_cache.ProjectId.Name), HttpUtility.UrlEncode(toolName), HttpUtility.UrlEncode(guid));
                // TODO: this would probably be faster if we directly called the RPC socket if FW is already open
                if (MiscUtils.IsUnix)
                {
                    string libPath = Path.GetDirectoryName(FileUtils.StripFilePrefix(Assembly.GetExecutingAssembly().CodeBase));
                    using (Process.Start(Path.Combine(libPath, "run-app"), string.Format("FieldWorks.exe {0}", url))) {}
                }
                else
                {
                    using (Process.Start(url)) {}
                }
            }
        }
Beispiel #4
0
        public void TermWithField()
        {
            QueryLexer lexer = Lex("title:foo");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme fieldLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Field, fieldLexeme.Type);
            Assert.Equal("title", fieldLexeme.Value);
            Assert.Equal(0, fieldLexeme.Start);
            Assert.Equal(5, fieldLexeme.End);

            Lexeme termLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Term, termLexeme.Type);
            Assert.Equal("foo", termLexeme.Value);
            Assert.Equal(6, termLexeme.Start);
            Assert.Equal(9, termLexeme.End);
        }
Beispiel #5
0
        private bool Take(LexemeKind lexemeKind, out Lexeme lexeme, bool ignoreWhitespace = true)
        {
            Push();

            if (ignoreWhitespace)
            {
                SkipWhitespaces();
            }

            if (Current.Kind == lexemeKind)
            {
                lexeme = TakeAny();
                Pop(false);
                return(true);
            }

            lexeme = null;
            Pop();
            return(false);
        }
Beispiel #6
0
        public void MultipleCreatesReferToSameSenses()
        {
            Lexeme lex  = m_lexicon.CreateLexeme(LexemeType.Word, "a");
            Lexeme lex2 = m_lexicon.CreateLexeme(LexemeType.Word, "a");

            m_lexicon.AddLexeme(lex);
            LexiconSense sense = lex.AddSense();

            sense.AddGloss("en", "test");

            Assert.AreEqual(1, lex2.Senses.Count());

            // Make sure the one that was added has the right sense now
            lex = m_lexicon[lex.Id];
            Assert.AreEqual(LexemeType.Word, lex.Type);
            Assert.AreEqual("a", lex.LexicalForm);
            Assert.AreEqual(1, lex.Senses.Count());
            Assert.AreEqual("en", lex.Senses.First().Glosses.First().Language);
            Assert.AreEqual("test", lex.Senses.First().Glosses.First().Text);
        }
Beispiel #7
0
        private static bool parserDefinition()
        {
            current_lexeme = Lexer.getDefinitionLexeme();
            switch (current_lexeme)
            {
            case Lexeme.NAME:
                if (!withdrawLexem(Lexeme.NAME) ||
                    !withdrawLexem(Lexeme.COLON) ||
                    !parserValue() ||
                    !parserProperties())
                {
                    return(false);
                }
                break;

            default:
                return(true);
            }
            return(parserDefinition());
        }
Beispiel #8
0
        public void RemoveSenseSucceeds()
        {
            Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Word, "a");

            m_lexicon.AddLexeme(lex);

            LexiconSense sense = lex.AddSense();

            sense.AddGloss("en", "gloss1");

            LexiconSense sense2 = lex.AddSense();

            sense.AddGloss("en", "gloss1");

            // Test remove at
            lex.RemoveSense(sense2);

            Assert.AreEqual(1, lex.Senses.Count());
            Assert.AreEqual(sense, lex.Senses.First());
        }
Beispiel #9
0
        public void MultipleTermsProduceTwoLexemes()
        {
            QueryLexer lexer = Lex("foo bar");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme fooLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Term, fooLexeme.Type);
            Assert.Equal("foo", fooLexeme.Value);
            Assert.Equal(0, fooLexeme.Start);
            Assert.Equal(3, fooLexeme.End);

            Lexeme barLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Term, barLexeme.Type);
            Assert.Equal("bar", barLexeme.Value);
            Assert.Equal(4, barLexeme.Start);
            Assert.Equal(7, barLexeme.End);
        }
Beispiel #10
0
        public void TermWithBoost()
        {
            QueryLexer lexer = Lex("foo^10");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme termLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Term, termLexeme.Type);
            Assert.Equal("foo", termLexeme.Value);
            Assert.Equal(0, termLexeme.Start);
            Assert.Equal(3, termLexeme.End);

            Lexeme boostLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Boost, boostLexeme.Type);
            Assert.Equal("10", boostLexeme.Value);
            Assert.Equal(4, boostLexeme.Start);
            Assert.Equal(6, boostLexeme.End);
        }
Beispiel #11
0
        public void TermWithEditDistance()
        {
            QueryLexer lexer = Lex("foo~2");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme termLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Term, termLexeme.Type);
            Assert.Equal("foo", termLexeme.Value);
            Assert.Equal(0, termLexeme.Start);
            Assert.Equal(3, termLexeme.End);

            Lexeme editDistanceLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.EditDistance, editDistanceLexeme.Type);
            Assert.Equal("2", editDistanceLexeme.Value);
            Assert.Equal(4, editDistanceLexeme.Start);
            Assert.Equal(5, editDistanceLexeme.End);
        }
Beispiel #12
0
        public void TermWithPresenceProhibited()
        {
            QueryLexer lexer = Lex("-foo");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme presenceLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Presence, presenceLexeme.Type);
            Assert.Equal("-", presenceLexeme.Value);
            Assert.Equal(0, presenceLexeme.Start);
            Assert.Equal(1, presenceLexeme.End);

            Lexeme termLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Term, termLexeme.Type);
            Assert.Equal("foo", termLexeme.Value);
            Assert.Equal(1, termLexeme.Start);
            Assert.Equal(4, termLexeme.End);
        }
Beispiel #13
0
        public void SingleTermWithHyphenProducesTwoLexemes()
        {
            QueryLexer lexer = Lex("foo-bar");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme fooLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Term, fooLexeme.Type);
            Assert.Equal("foo", fooLexeme.Value);
            Assert.Equal(0, fooLexeme.Start);
            Assert.Equal(3, fooLexeme.End);

            Lexeme barLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Term, barLexeme.Type);
            Assert.Equal("bar", barLexeme.Value);
            Assert.Equal(4, barLexeme.Start);
            Assert.Equal(7, barLexeme.End);
        }
Beispiel #14
0
        public void TermWithFieldWithEscapeChar()
        {
            QueryLexer lexer = Lex(@"ti\:tle:foo");

            Assert.Equal(2, lexer.Lexemes.Count);

            Lexeme fieldLexeme = lexer.Lexemes[0];

            Assert.Equal(LexemeType.Field, fieldLexeme.Type);
            Assert.Equal("ti:tle", fieldLexeme.Value);
            Assert.Equal(0, fieldLexeme.Start);
            Assert.Equal(7, fieldLexeme.End);

            Lexeme termLexeme = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Term, termLexeme.Type);
            Assert.Equal("foo", termLexeme.Value);
            Assert.Equal(8, termLexeme.Start);
            Assert.Equal(11, termLexeme.End);
        }
Beispiel #15
0
        private VariableInfo ResolveVariable(Lexeme identifier)
        {
            //step1, check local parameter & variable definitions
            if (m_currentMethodParameters.Contains(identifier.Value))
            {
                return(m_currentMethodParameters[identifier.Value]);
            }
            else if (m_currentMethodVariables.Contains(identifier.Value))
            {
                return(m_currentMethodVariables[identifier.Value]);
            }
            //step2, if not static method, check fields
            if (!m_currentMethod.IsStatic)
            {
                return(ResolveField(m_currentType, identifier));
            }

            m_errorManager.AddError(c_SE_VariableDeclMissing, identifier.Span, identifier.Value);
            return(null);
        }
Beispiel #16
0
        protected void processRightParenthesisLexeme(Lexeme rightParen)
        {
            while (true)
            {
                Lexeme popped;
                try
                {
                    popped = operatorStack.Pop();
                }
                catch (InvalidOperationException ex)
                {
                    throw new ParseException("Mismatched parentheses!");
                }

                if (popped.Type == Lexeme.LexemeType.LEFT_PAREN)
                {
                    try
                    {
                        // Ako je na vrhu stoga ostalo ime funkcije, prebacujemo ga u izlaz
                        IToken stackTop = operatorStack.Peek();
                        if (stackTop is Lexeme && properties.IsFunctionName((stackTop as Lexeme).Value))
                        {
                            Lexeme funcName = stackTop as Lexeme;
                            output.Enqueue(funcName);
                            operatorStack.Pop();
                            openedArgumentLists--;
                        }
                    }
                    catch (InvalidOperationException ex)
                    {
                        // do nothing, operator stack remains empty
                    }

                    return;
                }
                else
                {
                    output.Enqueue(popped);
                }
            }
        }
Beispiel #17
0
        public override Parse <TFuture> BuildParse <TFuture>(Future <string, TFuture> future)
        {
            Parse <TFuture> parse = null;

            parse = scanner =>
            {
                Lexeme l = scanner.Read();

                if (l.IsEndOfStream)
                {
                    return(new StepResult <TFuture>(0, () => future(l.Value.Content)(scanner)));
                }

                return(new StepResult <TFuture>(
                           1,
                           () => parse(scanner),
                           Grammar.RecoverByDeletion(l)));
            };

            return(parse);
        }
Beispiel #18
0
        /********************************************************************
        *** FUNCTION ProcessDoubleToken                                   ***
        *********************************************************************
        *** DESCRIPTION : This function is reponsible to categorize all the**
        *** double operators like relational and arithmetic operators     ***
        *** INPUT ARGS : -                                                ***
        *** OUTPUT ARGS : -                                               ***
        *** IN/OUT ARGS : -                                               ***
        *** RETURN : void                                                 ***
        ********************************************************************/
        private static void ProcessDoubleToken()
        {
            Lexeme = string.Concat(Lexeme, ch);

            if (Lexeme.Equals("==") || Lexeme.Equals("!=") || Lexeme.Equals("<=") || Lexeme.Equals(">="))
            {
                Token = Symbol.relop;
            }

            if (Lexeme.Equals("||"))
            {
                Token = Symbol.addop;
            }

            if (Lexeme.Equals("&&"))
            {
                Token = Symbol.mulop;
            }

            GetNextCh();
        }
Beispiel #19
0
        public void MorphTypeRetained()
        {
            Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Word, "a");

            m_lexicon.AddLexeme(lex);
            Lexeme lex2 = m_lexicon.CreateLexeme(LexemeType.Prefix, "a");

            m_lexicon.AddLexeme(lex2);
            Lexeme lex3 = m_lexicon.CreateLexeme(LexemeType.Suffix, "a");

            m_lexicon.AddLexeme(lex3);
            Lexeme lex4 = m_lexicon.CreateLexeme(LexemeType.Stem, "a");

            m_lexicon.AddLexeme(lex4);

            Assert.AreEqual(4, m_lexicon.Lexemes.Count());
            Assert.IsTrue(m_lexicon.Lexemes.Contains(lex));
            Assert.IsTrue(m_lexicon.Lexemes.Contains(lex2));
            Assert.IsTrue(m_lexicon.Lexemes.Contains(lex3));
            Assert.IsTrue(m_lexicon.Lexemes.Contains(lex4));
        }
Beispiel #20
0
        public void NormalizeStrings()
        {
            Lexeme lex = m_lexicon.CreateLexeme(LexemeType.Stem, "Vacaci\u00f3n");             // Uses composed accented letter 'o'

            m_lexicon.AddLexeme(lex);

            lex = m_lexicon[new LexemeKey(LexemeType.Stem, "Vacaci\u00f3n").Id];
            Assert.IsNotNull(lex);
            Assert.AreEqual(LexemeType.Stem, lex.Type);
            Assert.AreEqual("Vacaci\u00f3n", lex.LexicalForm);

            LexiconSense sense = lex.AddSense();

            Assert.IsNotNull(sense);

            LanguageText gloss = sense.AddGloss("en", "D\u00f3nde");

            Lexeme reGetLex = m_lexicon[lex.Id];

            Assert.AreEqual(gloss.Text, reGetLex.Senses.First().Glosses.First().Text);
        }
Beispiel #21
0
 public Token ProcessState(InputStream pInput, Lexeme pLexeme)
 {
     if (char.IsDigit(pInput.CurrentSymbol))
     {
         pLexeme.addSymbol(pInput.CurrentSymbol);
         pInput.ConsumeSymbol();
         return(new StateDigit().ProcessState(pInput, pLexeme));
     }
     else if (pInput.CurrentSymbol == '.')
     {
         pLexeme.addSymbol(pInput.CurrentSymbol);
         pInput.ConsumeSymbol();
         return(new StateFloat1().ProcessState(pInput, pLexeme));
     }
     else
     {
         return(new Token {
             Type = TokenType.LitInteger, LexemeVal = pLexeme.Value,
             Column = pInput.Column, Row = pInput.Row
         });
     }
 }
Beispiel #22
0
        public void MultipleTermsWithPresence()
        {
            QueryLexer lexer = Lex("+foo +bar");

            Assert.Equal(4, lexer.Lexemes.Count);

            Lexeme fooPresenceLexeme = lexer.Lexemes[0];
            Lexeme fooLexeme         = lexer.Lexemes[1];

            Assert.Equal(LexemeType.Presence, fooPresenceLexeme.Type);
            Assert.Equal(LexemeType.Term, fooLexeme.Type);
            Assert.Equal("+", fooPresenceLexeme.Value);
            Assert.Equal("foo", fooLexeme.Value);

            Lexeme barPresenceLexeme = lexer.Lexemes[2];
            Lexeme barLexeme         = lexer.Lexemes[3];

            Assert.Equal(LexemeType.Presence, barPresenceLexeme.Type);
            Assert.Equal(LexemeType.Term, barLexeme.Type);
            Assert.Equal("+", barPresenceLexeme.Value);
            Assert.Equal("bar", barLexeme.Value);
        }
Beispiel #23
0
        private static bool parserProperties()
        {
            current_lexeme = Lexer.getPropertiesLexeme();
            switch (current_lexeme)
            {
            case Lexeme.SEMICOLON:
                if (!withdrawLexem(Lexeme.SEMICOLON))
                {
                    return(false);
                }
                return(true);

            default:
                if (Lexer.checkForWhitespace() ||
                    !parserValue() ||
                    !parserProperties())
                {
                    return(false);
                }
                return(true);
            }
        }
Beispiel #24
0
        public void UpdateLexim()
        {
            IThemeGateway themeGateway = Gateway.ThemeGateway;

            themeGateway.DeleteAllThemes();
            Theme theme = new Theme(0, "Animals");
            int   id    = themeGateway.AddTheme(theme);

            ILexemGateway lexemGateway = Gateway.LexemGateway;
            Lexeme        lex          = Lexemes.LexemeCollection[0];

            lex.ParentThemeID = id;
            lex.ID            = lexemGateway.AddLexeme(lex);

            lex.Word = "Fossil";
            lexemGateway.UpdateLexeme(lex);

            Lexeme lex1 = lexemGateway.GetLexeme(lex.ID);

            Assert.AreEqual("Fossil", lex1.Word);
            Assert.AreEqual(lex.ID, lex1.ID);
        }
Beispiel #25
0
 void ThemeView_OnSaveLexim(LeximDTView leximDT)
 {
     try
     {
         Lexeme lexim = new Lexeme(leximDT.ID, leximDT.ParentThemeID, leximDT.Word, leximDT.Picture, leximDT.Sound);
         if (lexim.ID == 0)
         {
             leximDT.ID = LexemGateway.AddLexeme(lexim);
             this.ThemeView.LeximListBox.InsertLexim(0, leximDT, this.ThemeView.Play);
             this.ThemeView.LeximListBox.SetSelectionAt(0);
         }
         else
         {
             LexemGateway.UpdateLexeme(lexim);
             this.ThemeView.LeximListBox.UpdateLexim(leximDT);
         }
     }
     catch (Exception ex)
     {
         this.HandleException("Could not add or update a word ", ex);
     }
 }
Beispiel #26
0
        private Expression Assignment()
        {
            Expression expr = Or();

            if (Match(LexemeType.EQUAL))
            {
                Lexeme     equals = Previous();
                Expression value  = Assignment();

                if (expr is Expression.Variable &&
                    ((Expression.Variable)expr) != null &&
                    ((Expression.Variable)expr).Name != null)
                {
                    Expression.Variable variable = (Expression.Variable)expr;
                    if (variable != null && variable.Name != null)
                    {
                        return(new Expression.Assign(variable.Name, value));
                    }
                    //> Classes assign-set
                }

                if (expr is Expression.Get &&
                    ((Expression.Get)expr) != null &&
                    ((Expression.Get)expr).Name != null)
                {
                    Expression.Get get = (Expression.Get)expr;
                    if (get != null && get.Name != null)
                    {
                        return(new Expression.Set(get, get.Name, value));
                    }
                    //< Classes assign-set
                }

                //error(equals, "Invalid assignment target."); // [no-throw]
            }

            return(expr);
        }
Beispiel #27
0
        public override Parse <TFuture> BuildParse <TFuture>(Future <string, TFuture> future)
        {
            Parse <TFuture> parse = null;

            parse = scanner =>
            {
                // 保存读取当前单词之前的位置
                ForkableScanner prevScanner = scanner.Fork();

                Lexeme l = scanner.Read();

                if (l.TokenIndex == m_token.Index)
                {
                    return(new StepResult <TFuture>(0, () => future(l.Value.Content)(scanner)));
                }

                Lexeme      recovery     = l.GetErrorCorrectionLexeme(m_token.Index, m_token.Description);
                SyntaxError insertionErr = Grammar.RecoverByInsertion(recovery);

                if (l.IsEndOfStream)
                {
                    // 已经到了输入的末尾
                    // 插入预期的Token进行恢复
                    return(new StepResult <TFuture>(1,
                                                    () => future(recovery.Value.Content)(prevScanner), insertionErr));
                }
                else
                {
                    // 同时尝试插入预期的Token,以及删除当前字符
                    // 在未来的解析中选取更好的路线
                    return(Grammar.Best(
                               new StepResult <TFuture>(1, () => future(recovery.Value.Content)(prevScanner), insertionErr),
                               new StepResult <TFuture>(1, () => parse(scanner), Grammar.RecoverByDeletion(l))));
                }
            };

            return(parse);
        }
Beispiel #28
0
        public static LinkedListNode <Lexeme> TryParse(LinkedListNode <Lexeme> lexemes, out FnParam resultNode)
        {
            resultNode = null;
            var nextLexeme = Type.TryParse(lexemes, out Type type);

            if (type == null)
            {
                return(lexemes);
            }

            if (nextLexeme.Value.Type != LT.IDENT)
            {
                throw new Exception($"Missing function argument name at {nextLexeme.Value.File}:{nextLexeme.Value.Line}");
            }

            Lexeme token = nextLexeme.Value;

            nextLexeme = nextLexeme.Next;

            if (nextLexeme.Value.Type == LT.OP_ASSIGN)
            {
                nextLexeme = nextLexeme.Next;
                nextLexeme = Literal.TryParse(nextLexeme, out Literal lit);
                if (lit == null)
                {
                    throw new Exception($"Missing default value after assignment operator in function arguments list at {nextLexeme.Value.File}:{nextLexeme.Value.Line}");
                }
                resultNode = new FnParam {
                    Type = type, DefaultValue = lit, Token = token
                };
                return(nextLexeme);
            }

            resultNode = new FnParam {
                Type = type, Token = token
            };
            return(nextLexeme);
        }
Beispiel #29
0
        private Expression FinishCall(Expression callee)
        {
            List <Expression> arguments = new List <Expression>();

            if (!Check(LexemeType.RIGHT_PAREN))
            {
                do
                {
                    ////> check-max-arity
                    //if (arguments.size() >= 255)
                    //{
                    //    error(peek(), "Can't have more than 255 arguments.");
                    //}
                    ////< check-max-arity

                    arguments.Add(Expr());
                } while (Match(LexemeType.COMMA));
            }

            Lexeme paren = Consume(LexemeType.RIGHT_PAREN, Peek());

            return(new Expression.Call(callee, paren, arguments));
        }
Beispiel #30
0
        //private bool isSubscriptedIdentifier(SubscriptToken t)
        //{
        //    TokenList subBase = (token as SubscriptToken).Base;
        //    TokenList subscript = (token as SubscriptToken).Subscript;

        //    if (subBase.Count == 1 && subBase[0] is TextRunToken)
        //    {
        //        String subBaseText = (subBase[0] as TextRunToken).Text;
        //        if (Regex.Match(subBaseText, )
        //            }
        //    // TODO : implementiraj!!!!
        //    return false;
        //}

        protected bool canProcessTokenAsUnaryOp()
        {
            if (lastProcessedElement == null)
            {
                return(true);
            }
            else if (lastProcessedElement is Lexeme)
            {
                Lexeme            previous = lastProcessedElement as Lexeme;
                Lexeme.LexemeType type     = previous.Type;
                return(type == Lexeme.LexemeType.LEFT_PAREN ||
                       type == Lexeme.LexemeType.EQ_SIGN ||
                       type == Lexeme.LexemeType.OP_DIV ||
                       type == Lexeme.LexemeType.OP_MUL ||
                       type == Lexeme.LexemeType.OP_MINUS ||
                       type == Lexeme.LexemeType.OP_PLUS ||
                       type == Lexeme.LexemeType.ARGUMENT_SEPARATOR);
            }
            else
            {
                return(false);
            }
        }
        /// <summary>*
        /// 处理数字字母混合输出
        /// 如:windos2000 | [email protected]
        /// </summary>
        private bool ProcessMixLetter(char input, Context context)
        {
            bool needLock = false;

            if (start == -1)
            {//当前的分词器尚未开始处理字符			
                if (IsAcceptedCharStart(input))
                {
                    //记录起始指针的位置,标明分词器进入处理状态
                    start = context.Cursor;
                    end = start;
                }

            }
            else
            {//当前的分词器正在处理字符			
                if (IsAcceptedChar(input))
                {
                    //输入不是连接符
                    //				if(!isLetterConnector(input)){
                    //记录下可能的结束位置,如果是连接符结尾,则忽略
                    //					end = context.getCursor();					
                    //				}
                    //不在忽略尾部的链接字符
                    end = context.Cursor;

                }
                else
                {
                    //生成已切分的词元
                    Lexeme newLexeme = new Lexeme(context.BuffOffset, start, end - start + 1, Lexeme.TYPE_LETTER);
                    context.AddLexeme(newLexeme);
                    //设置当前分词器状态为“待处理”
                    start = -1;
                    end = -1;
                }
            }

            //context.getCursor() == context.getAvailable() - 1读取缓冲区最后一个字符,直接输出
            if (context.Cursor == context.Available - 1)
            {
                if (start != -1 && end != -1)
                {
                    //生成已切分的词元
                    Lexeme newLexeme = new Lexeme(context.BuffOffset, start, end - start + 1, Lexeme.TYPE_LETTER);
                    context.AddLexeme(newLexeme);
                }
                //设置当前分词器状态为“待处理”
                start = -1;
                end = -1;
            }

            //判断是否锁定缓冲区
            if (start == -1 && end == -1)
            {
                //对缓冲区解锁
                needLock = false;
            }
            else
            {
                needLock = true;
            }
            return needLock;
        }
		public FdoLexiconGlossAddedEventArgs(Lexeme lexeme, LexiconSense sense, LanguageText gloss)
		{
			m_lexeme = lexeme;
			m_sense = sense;
			m_gloss = gloss;
		}
        public void CheckToken(Lexeme t)
        {
            Debug.Assert(Lexeme.Name <= t);  // FirstStringable
            if (lexeme != t)
            {
                if (t == Lexeme.Eof)
                    throw EofExpectedException(RawValue);

                else
                    throw TokenExpectedException(t, RawValue);
            }
        }
Beispiel #34
0
            /// <summary>
            /// 取出链表集合的最后一个元素
            /// </summary>
            /// <returns></returns>
            internal Lexeme PollLast()
            {
                if (this.size == 1)
                {
                    Lexeme last = this.head;
                    this.head = null;
                    this.tail = null;
                    this.size--;
                    return last;

                }
                else if (this.size > 1)
                {
                    Lexeme last = this.tail;
                    this.tail = last.Prev;
                    last.Prev = null;
                    this.size--;
                    return last;

                }
                else
                {
                    return null;
                }
            }
Beispiel #35
0
            /// <summary>
            /// 向链表集合添加词元
            /// </summary>
            internal void AddLexeme(Lexeme lexeme)
            {
                if (this.size == 0)
                {
                    this.head = lexeme;
                    this.tail = lexeme;
                    this.size++;
                    return;

                }
                else
                {
                    if (this.tail.CompareTo(lexeme) == 0)
                    {//词元与尾部词元相同,不放入集合
                        return;

                    }
                    else if (this.tail.CompareTo(lexeme) < 0)
                    {//词元接入链表尾部
                        this.tail.Next = lexeme;
                        lexeme.Prev = this.tail;
                        this.tail = lexeme;
                        this.size++;
                        return;

                    }
                    else if (this.head.CompareTo(lexeme) > 0)
                    {//词元接入链表头部
                        this.head.Prev = lexeme;
                        lexeme.Next = this.head;
                        this.head = lexeme;
                        this.size++;
                        return;

                    }
                    else
                    {
                        //从尾部上逆
                        Lexeme l = this.tail;
                        while (l != null && l.CompareTo(lexeme) > 0)
                        {
                            l = l.Prev;
                        }
                        if (l.CompareTo(lexeme) == 0)
                        {//词元与集合中的词元重复,不放入集合
                            return;

                        }
                        else if (l.CompareTo(lexeme) < 0)
                        {//词元插入链表中的某个位置
                            lexeme.Prev = l;
                            lexeme.Next = l.Next;
                            l.Next.Prev = lexeme;
                            l.Next = lexeme;
                            this.size++;
                            return;

                        }
                    }
                }

            }
        /// <summary>
        /// 添加量词词元到结果集
        /// </summary>
        /// <param name="context"></param>
        private void OutputCountLexeme(Context context)
        {
            if (countStart > -1 && countEnd > -1)
            {
                //生成已切分的词元
                Lexeme countLexeme = new Lexeme(context.BuffOffset, countStart, countEnd - countStart + 1, Lexeme.TYPE_NUMCOUNT);
                context.AddLexeme(countLexeme);
            }

        }
        public void NextLex()
        {
            prevLexEnd = curIndex;
            prevLexeme = lexeme;
            SkipWhiteSpace();
            lexStart = curIndex;

            switch (curChar)
            {
                case '\0':
                    lexeme = Lexeme.Eof;
                    return;
                case '(':
                case ')':
                case '[':
                case ']':
                case '@':
                case ',':
                case '$':
                case '}':
                    lexeme = (Lexeme)curChar;
                    NextChar();
                    break;
                case '.':
                    NextChar();
                    if (curChar == '.')
                    {
                        lexeme = Lexeme.DotDot;
                        NextChar();
                    }
                    else if (IsAsciiDigit(curChar))
                    {
                        SetSourceIndex(lexStart);
                        goto case '0';
                    }
                    else
                        lexeme = Lexeme.Dot;

                    break;
                case ':':
                    NextChar();
                    if (curChar == ':')
                    {
                        lexeme = Lexeme.ColonColon;
                        NextChar();
                    }
                    else
                    {
                        lexeme = Lexeme.Unknown;
                    }
                    break;
                case '*':
                    lexeme = Lexeme.Star;
                    NextChar();
                    CheckOperator(true);
                    break;
                case '/':
                    NextChar();
                    if (curChar == '/')
                    {
                        lexeme = Lexeme.SlashSlash;
                        NextChar();
                    }
                    else
                    {
                        lexeme = Lexeme.Slash;
                    }
                    break;
                case '|':
                    lexeme = Lexeme.Union;
                    NextChar();
                    break;
                case '+':
                    lexeme = Lexeme.Plus;
                    NextChar();
                    break;
                case '-':
                    lexeme = Lexeme.Minus;
                    NextChar();
                    break;
                case '=':
                    lexeme = Lexeme.Eq;
                    NextChar();
                    break;
                case '!':
                    NextChar();
                    if (curChar == '=')
                    {
                        lexeme = Lexeme.Ne;
                        NextChar();
                    }
                    else
                    {
                        lexeme = Lexeme.Unknown;
                    }
                    break;
                case '<':
                    NextChar();
                    if (curChar == '=')
                    {
                        lexeme = Lexeme.Le;
                        NextChar();
                    }
                    else
                    {
                        lexeme = Lexeme.Lt;
                    }
                    break;
                case '>':
                    NextChar();
                    if (curChar == '=')
                    {
                        lexeme = Lexeme.Ge;
                        NextChar();
                    }
                    else
                    {
                        lexeme = Lexeme.Gt;
                    }
                    break;
                case '"':
                case '\'':
                    lexeme = Lexeme.String;
                    ScanString();
                    break;
                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                    lexeme = Lexeme.Number;
                    ScanNumber();
                    break;
                default:
                    this.name = ScanNCName();
                    if (this.name != null)
                    {
                        lexeme = Lexeme.Name;
                        this.prefix = string.Empty;
                        this.canBeFunction = false;
                        this.axis = XPathAxis.Unknown;
                        bool colonColon = false;
                        int saveSourceIndex = curIndex;

                        // "foo:bar" or "foo:*" -- one lexeme (no spaces allowed)
                        // "foo::" or "foo ::"  -- two lexemes, reported as one (AxisName)
                        // "foo:?" or "foo :?"  -- lexeme "foo" reported
                        if (curChar == ':')
                        {
                            NextChar();
                            if (curChar == ':')
                            {   // "foo::" -> OperatorName, AxisName
                                NextChar();
                                colonColon = true;
                                SetSourceIndex(saveSourceIndex);
                            }
                            else
                            {                // "foo:bar", "foo:*" or "foo:?"
                                string ncName = ScanNCName();
                                if (ncName != null)
                                {
                                    this.prefix = this.name;
                                    this.name = ncName;
                                    // Look ahead for '(' to determine whether QName can be a FunctionName
                                    saveSourceIndex = curIndex;
                                    SkipWhiteSpace();
                                    this.canBeFunction = (curChar == '(');
                                    SetSourceIndex(saveSourceIndex);
                                }
                                else if (curChar == '*')
                                {
                                    NextChar();
                                    this.prefix = this.name;
                                    this.name = "*";
                                }
                                else
                                {            // "foo:?" -> OperatorName, NameTest
                                    // Return "foo" and leave ":" to be reported later as an unknown lexeme
                                    SetSourceIndex(saveSourceIndex);
                                }
                            }
                        }
                        else
                        {
                            SkipWhiteSpace();
                            if (curChar == ':')
                            {   // "foo ::" or "foo :?"
                                NextChar();
                                if (curChar == ':')
                                {
                                    NextChar();
                                    colonColon = true;
                                }
                                SetSourceIndex(saveSourceIndex);
                            }
                            else
                            {
                                this.canBeFunction = (curChar == '(');
                            }
                        }
                        if (!CheckOperator(false) && colonColon)
                        {
                            this.axis = CheckAxis();
                        }
                    }
                    else
                    {
                        lexeme = Lexeme.Unknown;
                        NextChar();
                    }
                    break;
            }
        }
 private List<Lexeme> AllowedFollowing(Lexeme t)
 {
     var list = new List<Lexeme>();
     switch (t)
     {
         case Lexeme.Axis:
             list.Add(Lexeme.ColonColon);
             return list;
         case Lexeme.ColonColon:
             list.Add(Lexeme.Name);
             list.Add(Lexeme.Star);
             return list;
         case Lexeme.And:
             list.Add(Lexeme.String);
             return list;
         case Lexeme.At:
             list.Add(Lexeme.Name);
             list.Add(Lexeme.Star);
             return list;
         case Lexeme.Name:
             list.Add(Lexeme.Slash);
             list.Add(Lexeme.LParens);
             list.Add(Lexeme.LBracket);
             return list;
         case Lexeme.Slash:
             list.Add(Lexeme.Name);
             list.Add(Lexeme.Star);
             return list;
         case Lexeme.SlashSlash:
             list.Add(Lexeme.Name);
             list.Add(Lexeme.Star);
             return list;
         case Lexeme.Eq:
             list.Add(Lexeme.Number);
             list.Add(Lexeme.Name);
             return list;
         case Lexeme.LParens:
             list.Add(Lexeme.RParens);
             list.Add(Lexeme.Name);
             return list;
     }
     return list;
 }
 private List<Lexeme> AllowedWhenExpected(Lexeme t)
 {
     var list = new List<Lexeme>();
     list.Add(t);
     switch (t)
     {
         case Lexeme.RBracket:
             {
                 bool foundEq= false;
                 // last on the stack is first in the array
                 var x = lexemeStack.ToArray();
                 for (int i= 0; i < x.Length; i++)
                 {
                     if (x[i] == Lexeme.LBracket) break;
                     if (x[i] == Lexeme.Eq) { foundEq = true; break; }
                 }
                 if (!foundEq)
                     list.Add(Lexeme.Eq);
             }
             break;
     }
     return list;
 }
 private XPathAxis CheckAxis()
 {
     this.lexeme = Lexeme.Axis;
     switch (name)
     {
         case "ancestor": return XPathAxis.Ancestor;
         case "ancestor-or-self": return XPathAxis.AncestorOrSelf;
         case "attribute": return XPathAxis.Attribute;
         case "child": return XPathAxis.Child;
         case "descendant": return XPathAxis.Descendant;
         case "descendant-or-self": return XPathAxis.DescendantOrSelf;
         case "following": return XPathAxis.Following;
         case "following-sibling": return XPathAxis.FollowingSibling;
         case "namespace": return XPathAxis.Namespace;
         case "parent": return XPathAxis.Parent;
         case "preceding": return XPathAxis.Preceding;
         case "preceding-sibling": return XPathAxis.PrecedingSibling;
         case "self": return XPathAxis.Self;
         default:
             this.lexeme = Lexeme.Name;
             return XPathAxis.Unknown;
     }
 }
        private bool CheckOperator(bool star)
        {
            Lexeme op;

            if (star)
            {
                op = Lexeme.Multiply;
            }
            else
            {
                if (prefix.Length != 0 || name.Length > 3)
                    return false;

                switch (name)
                {
                    case "or": op = Lexeme.Or; break;
                    case "and": op = Lexeme.And; break;
                    case "div": op = Lexeme.Divide; break;
                    case "mod": op = Lexeme.Modulo; break;
                    default: return false;
                }
            }

            // If there is a preceding token and the preceding token is not one of '@', '::', '(', '[', ',' or an Operator,
            // then a '*' must be recognized as a MultiplyOperator and an NCName must be recognized as an OperatorName.
            if (prevLexeme <= Lexeme.LastOperator)
                return false;

            switch (prevLexeme)
            {
                case Lexeme.Slash:
                case Lexeme.SlashSlash:
                case Lexeme.At:
                case Lexeme.ColonColon:
                case Lexeme.LParens:
                case Lexeme.LBracket:
                case Lexeme.Comma:
                case Lexeme.Dollar:
                    return false;
            }

            this.lexeme = op;
            return true;
        }
		public ExpressionLexeme(string lexemeBody, Lexeme lexeme)
		{
			_lexemeBody = lexemeBody;
			_lexeme = lexeme;
		}
		private static LexemeType GetLexemeType(Lexeme lexeme)
		{			
			if (ExpressionLexemeInfo.LexemeTypes.ContainsKey(lexeme))
				return (LexemeType)ExpressionLexemeInfo.LexemeTypes[lexeme];
			else
				return LexemeType.ltUndefined;
		}
        // May be called for the following tokens: Name, String, Eof, Comma, LParens, RParens, LBracket, RBracket, RBrace
        private string LexemeToString(Lexeme t)
        {
            Debug.Assert(Lexeme.Name <= t);

            if (Lexeme.Eof < t)
            {
                Debug.Assert("()[].@,*/$}".IndexOf((char)t) >= 0);
                return new string((char)t, 1);
            }

            switch (t)
            {
                case Lexeme.Name: return "<name>";
                case Lexeme.String: return "<string literal>";
                case Lexeme.Eof: return "<eof>";
                default:
                    Debug.Fail("Unexpected Lexeme: " + t.ToString());
                    return string.Empty;
            }
        }
 public XPathParserException TokenExpectedException(Lexeme t, string actualToken)
 {
     string expectedToken = LexemeToString(t);
     var x = new XPathParserException(xpathExpr, lexStart, curIndex,
         string.Format("Expected token '{0}', found '{1}'.", expectedToken, actualToken)
     );
     x.NextAllowed= AllowedWhenExpected(t);
     return x;
 }
Beispiel #46
0
 public Rule(String regex, Lexeme.Type type)
     : this(regex, DefaultProducer(type))
 {
 }
 /// <summary>
 /// 添加数词词元到结果集
 /// </summary>
 /// <param name="context"></param>
 private void OutputNumLexeme(Context context)
 {
     if (nStart > -1 && nEnd > -1)
     {
         //生成已切分的词元
         Lexeme newLexeme = new Lexeme(context.BuffOffset, nStart, nEnd - nStart + 1, Lexeme.TYPE_NUM);
         context.AddLexeme(newLexeme);
         fCaN = true;
     }
 }
Beispiel #48
0
 private static Producer DefaultProducer(Lexeme.Type type)
 {
     return (value, pos) => new Lexeme(value, type, pos);
 }
Beispiel #49
0
 /// <summary>
 /// 向分词结果集添加词元
 /// </summary>
 /// <param name="lexeme"></param>
 public void AddLexeme(Lexeme lexeme)
 {
     if (!Dictionary.IsStopWord(segmentBuff, lexeme.Begin, lexeme.Length))
     {
         this.lexemeSet.AddLexeme(lexeme);
     }
 }
        /// <summary>
        /// 取出词元集合中的下一个词元
        /// </summary>
        /// <param name="lexeme"></param>
        /// <returns></returns>
        private Lexeme BuildLexeme(Lexeme lexeme)
        {
            if (lexeme != null)
            {
                // TODO:从数组取一段内容
                var arr = new char[lexeme.Length];
                Array.Copy(segmentBuff, lexeme.Begin, arr, 0, lexeme.Length);
                //生成lexeme的词元文本
                lexeme.LexemeText = string.Concat(arr);
                return lexeme;

            }
            else
            {
                return null;
            }
        }
Beispiel #51
0
 /// <summary>
 /// 取出链表集合的第一个元素
 /// </summary>
 internal Lexeme PollFirst()
 {
     if (this.size == 1)
     {
         Lexeme first = this.head;
         this.head = null;
         this.tail = null;
         this.size--;
         return first;
     }
     else if (this.size > 1)
     {
         Lexeme first = this.head;
         this.head = first.Next;
         first.Next = null;
         this.size--;
         return first;
     }
     else
     {
         return null;
     }
 }
Beispiel #52
0
 public TokenBranch(Lexeme lexeme)
 {
     if (lexeme != null)
     {
         this.lexeme = lexeme;
         //初始化branch的左右边界
         this.leftBorder = lexeme.BeginPosition;
         this.rightBorder = lexeme.EndPosition;
     }
 }
 /// <summary>
 /// 向分词结果集添加词元(尚未完成)
 /// </summary>
 /// <param name="lexeme"></param>
 public void AddLexeme(Lexeme lexeme)
 {
     
 }
Beispiel #54
0
            ///<summary>
            ///组合词元分支
            /// </summary>
            /// <returns>返回当前branch能否接收词元对象</returns>
            public bool Accept(Lexeme _lexeme)
            {

                /*
                 * 检查新的lexeme 对当前的branch 的可接受类型
                 * acceptType : REFUSED  不能接受
                 * acceptType : ACCEPTED 接受
                 * acceptType : TONEXT   由相邻分支接受 
                 */
                int acceptType = CheckAccept(_lexeme);
                switch (acceptType)
                {
                    case REFUSED:
                        // REFUSE 情况
                        return false;

                    case ACCEPTED:
                        if (acceptedBranchs == null)
                        {
                            //当前branch没有子branch,则添加到当前branch下
                            acceptedBranchs = new List<TokenBranch>(2);
                            acceptedBranchs.Add(new TokenBranch(_lexeme));
                        }
                        else
                        {
                            bool acceptedByChild = false;
                            //当前branch拥有子branch,则优先由子branch接纳
                            foreach (TokenBranch childBranch in acceptedBranchs)
                            {
                                acceptedByChild = childBranch.Accept(_lexeme) || acceptedByChild;
                            }
                            //如果所有的子branch不能接纳,则由当前branch接纳
                            if (!acceptedByChild)
                            {
                                acceptedBranchs.Add(new TokenBranch(_lexeme));
                            }
                        }
                        //设置branch的最大右边界
                        if (_lexeme.EndPosition > this.rightBorder)
                        {
                            this.rightBorder = _lexeme.EndPosition;
                        }
                        break;

                    case TONEXT:
                        //把lexeme放入当前branch的相邻分支
                        if (this.nextBranch == null)
                        {
                            //如果还没有相邻分支,则建立一个不交叠的分支
                            this.nextBranch = new TokenBranch(null);
                        }
                        this.nextBranch.Accept(_lexeme);
                        break;
                }

                return true;
            }
		public FdoLexemeAddedEventArgs(Lexeme lexeme)
		{
			m_lexeme = lexeme;
		}
Beispiel #56
0
            /// <summary>
            /// 判断指定的lexeme能否被当前的branch接受
            /// </summary>
            /// <param name="_lexeme"></param>
            /// <returns></returns>
            private int CheckAccept(Lexeme _lexeme)
            {
                int acceptType = 0;

                if (_lexeme == null)
                {
                    throw new ArgumentException("parameter:lexeme is null");
                }

                if (null == this.lexeme)
                {//当前的branch是一个不交叠(ROOT)的分支
                    if (this.rightBorder > 0  //说明当前branch内至少有一个lexeme
                            && _lexeme.BeginPosition >= this.rightBorder)
                    {
                        //_lexeme 与 当前的branch不相交
                        acceptType = TONEXT;
                    }
                    else
                    {
                        acceptType = ACCEPTED;
                    }
                }
                else
                {//当前的branch是一个有交叠的分支

                    if (_lexeme.BeginPosition < this.lexeme.BeginPosition)
                    {
                        //_lexeme 的位置比 this.lexeme还靠前(这种情况不应该发生)
                        acceptType = REFUSED;
                    }
                    else if (_lexeme.BeginPosition >= this.lexeme.BeginPosition
                               && _lexeme.BeginPosition < this.lexeme.EndPosition)
                    {
                        // _lexeme 与 this.lexeme相交
                        acceptType = REFUSED;
                    }
                    else if (_lexeme.BeginPosition >= this.lexeme.EndPosition
                               && _lexeme.BeginPosition < this.rightBorder)
                    {
                        //_lexeme 与 this.lexeme 不相交, 但_lexeme 与 当前的branch相交
                        acceptType = ACCEPTED;
                    }
                    else
                    {//_lexeme.BeginPosition >= this.rightBorder
                        //_lexeme 与 当前的branch不相交
                        acceptType = TONEXT;
                    }
                }
                return acceptType;
            }
		public FdoLexicalRelation(Lexeme lexeme, string name)
		{
			m_lexeme = lexeme;
			m_name = name;
		}
 public void PassToken(Lexeme t)
 {
     CheckToken(t);
     NextLex();
 }
		private static string GetLexemeDiffMask(Lexeme lexeme)
		{			
			return (string)ExpressionLexemeInfo.LexemeDiffMasks[lexeme];
		}
		private static string GetLexemeCode(Lexeme lexeme)
		{			
			return (string)ExpressionLexemeInfo.LexemeCodes[lexeme];
		}