Esempio n. 1
0
        public void NextTokenIgnoreNoiseTest()
        {
            string input;
            GrammarTokenizer target;
            Token token;

            input = "!* block comment \r\n*!<nonTerminal>";
            target = new GrammarTokenizer (input);
            token = target.NextTokenIgnoreNoise ();
            Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);

            input = "!* block comment \r\n<nonTerminal>";
            target = new GrammarTokenizer (input);
            token = target.NextTokenIgnoreNoise ();
            Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
        }
Esempio n. 2
0
        public void EmptyInputTest()
        {
            string input;
            GrammarTokenizer target;
            Token token;

            input = String.Empty;
            target = new GrammarTokenizer (input);
            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);

            input = null;
            target = new GrammarTokenizer (input);
            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
        }
Esempio n. 3
0
        public void NextTokenTest()
        {
            string input;
            GrammarTokenizer target;
            Token token;

            input = "\0";
            target = new GrammarTokenizer (input);
            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);

            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);

            input = "%{set}";
            target = new GrammarTokenizer (input);
            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);
            token = target.NextToken ();
            Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);
        }
Esempio n. 4
0
        Context getContext(int index)
        {
            Context context = Context.Unknown;

            if (!isInComment (index))
            {
                string lineStr = getLineUpTo (index);
                //tokenize the line, but don't include the trigger character
                var tokenizer = new GrammarTokenizer (lineStr.Remove (lineStr.Length - 1));
                var firstToken = tokenizer.NextTokenIgnoreNoise ();
                var firstSymbolType = TokenUtil.GetDefinitionType (firstToken);

                if (TokenUtil.IsUserDefinitionToken (firstToken))
                {
                    Token secondToken = tokenizer.NextTokenIgnoreNoise ();
                    if (secondToken.Symbol.Name == "=")
                    {
                        if (firstSymbolType==DefinitionType.Terminal)
                            context = Context.TerminalDeclaration;
                        else if (firstSymbolType==DefinitionType.SetName)
                            context = Context.SetDeclaration;
                    }
                    else if (secondToken.Symbol.Name == "::=" && firstSymbolType==DefinitionType.NonTerminal)
                    {
                        context = Context.RuleDeclaration;
                    }
                }
                else if (firstToken.Symbol.Name == "|")
                {
                    context = Context.RuleDeclaration;
                }
                else if (firstToken.Symbol.Kind == SymbolKind.End)
                {
                    context= Context.NewDeclaration;
                }
            }
            return context;
        }
Esempio n. 5
0
        //checks that the given line string is inside a terminal literal
        static bool isTerminalLiteralStart(string lineStr)
        {
            var tokenizer = new GrammarTokenizer (lineStr);
            var tokenList = new List<Token> ();
            Token token;
            do
            {
                token = tokenizer.NextToken();
                tokenList.Add(token);
            } while(token.Symbol.Kind!=SymbolKind.End);

            //when we are inside a terminal literal, the sequence would end with these tokens:
            // ERROR,TERMINAL,END
            const int ExpectedTokenCount = 3;
            int lastIndex = tokenList.Count - 1;
            if (tokenList.Count >= ExpectedTokenCount)
            {
                return
                    tokenList [lastIndex - 2].Symbol.Kind == SymbolKind.Error &&
                    tokenList [lastIndex - 1].Symbol.Kind == SymbolKind.Terminal;
            }
            return false;
        }
Esempio n. 6
0
        void parse(string text)
        {
            var tokenizer = new GrammarTokenizer (text);
            var tempList = new List<Token> ();
            Token token=null;
            Token prev=null;

            while (!TokenUtil.IsEnd(token))
            {
                token = tokenizer.NextToken ();
                tokenList.Add (token);

                if (!TokenUtil.IsNoise (token))
                {
                    if (!TokenUtil.IsNewLine (token))
                    {
                        if (isDefinitionStart (token, prev) || TokenUtil.IsEnd (token))
                        {
                            addDefinitionFrom (tempList);
                            tempList.Clear ();
                        }
                        tempList.Add (token);
                    }
                    prev = token;
                }
            }
        }