public void NonEmptyGrammarNonEmptyText()
        {
            LexerGrammar <LexerState> grammar = new LexerGrammar <LexerState>(
                new List <LexerTokenRule <LexerState> >()
            {
                new LexerTokenRule <LexerState>(1, "Text", "[a-zA-Z0-9]*"),
                new LexerTokenRule <LexerState>(
                    2,
                    "NewLine",
                    "\n",
                    (LexerState state, string lexem) =>
                {
                    state.LineNumber++;
                    return(LexerRuleReturnDecision.ReturnToken);
                }),
            },
                new List <LexerDynamicRule>());

            Lexer <LexerState> lexer = new Lexer <LexerState>(grammar);
            var s      = new LexerState();
            var tokens = lexer.GetTokens("Line1\nLine2\n", s).Tokens.ToArray();

            Assert.Equal(5, tokens.Count());

            Assert.Equal(3, s.LineNumber);
        }
        public void EmptyGrammarNonEmptyText()
        {
            LexerGrammar <LexerState> grammar = new LexerGrammar <LexerState>(new List <LexerTokenRule <LexerState> >(), new List <LexerDynamicRule>());
            Lexer <LexerState>        lexer   = new Lexer <LexerState>(grammar);

            Assert.False(lexer.GetTokens("Line1\nLine2\n").IsValid);
        }
        public void EmptyGrammarEmptyText()
        {
            LexerGrammar <LexerState> grammar = new LexerGrammar <LexerState>(new List <LexerTokenRule <LexerState> >(), new List <LexerDynamicRule>());
            Lexer <LexerState>        lexer   = new Lexer <LexerState>(grammar);
            var tokens = lexer.GetTokens(string.Empty).Tokens;

            Assert.Single(tokens);
        }
예제 #4
0
        public LexerATNFactory(LexerGrammar g)
            : base(g)
        {
            // use codegen to get correct language templates for lexer commands
            string         language = g.GetOptionString("language");
            CodeGenerator  gen      = new CodeGenerator(g.tool, null, language);
            AbstractTarget target   = gen.GetTarget();

            codegenTemplates = target != null?target.GetTemplates() : null;
        }
예제 #5
0
        /** Given the raw AST of a grammar, create a grammar object
            associated with the AST. Once we have the grammar object, ensure
            that all nodes in tree referred to this grammar. Later, we will
            use it for error handling and generally knowing from where a rule
            comes from.
         */
        public virtual Grammar CreateGrammar(GrammarRootAST ast)
        {
            Grammar g;
            if (ast.grammarType == ANTLRParser.LEXER)
                g = new LexerGrammar(this, ast);
            else
                g = new Grammar(this, ast);

            // ensure each node has pointer to surrounding grammar
            GrammarTransformPipeline.SetGrammarPtr(g, ast);
            return g;
        }
예제 #6
0
        /** To process a grammar, we load all of its imported grammars into
         *  subordinate grammar objects. Then we merge the imported rules
         *  into the root grammar. If a root grammar is a combined grammar,
         *  we have to extract the implicit lexer. Once all this is done, we
         *  process the lexer first, if present, and then the parser grammar
         */
        public virtual void Process(Grammar g, bool gencode)
        {
            g.LoadImportedGrammars();

            GrammarTransformPipeline transform = new GrammarTransformPipeline(g, this);

            transform.Process();

            LexerGrammar   lexerg;
            GrammarRootAST lexerAST;

            if (g.ast != null && g.ast.grammarType == ANTLRParser.COMBINED &&
                !g.ast.hasErrors)
            {
                lexerAST = transform.ExtractImplicitLexer(g); // alters g.ast
                if (lexerAST != null)
                {
                    if (grammarOptions != null)
                    {
                        lexerAST.cmdLineOptions = grammarOptions;
                    }

                    lexerg                    = new LexerGrammar(this, lexerAST);
                    lexerg.fileName           = g.fileName;
                    lexerg.originalGrammar    = g;
                    g.implicitLexer           = lexerg;
                    lexerg.implicitLexerOwner = g;

                    int prevErrors = errMgr.GetNumErrors();
                    ProcessNonCombinedGrammar(lexerg, gencode);
                    if (errMgr.GetNumErrors() > prevErrors)
                    {
                        return;
                    }

                    //				System.out.println("lexer tokens="+lexerg.tokenNameToTypeMap);
                    //				System.out.println("lexer strings="+lexerg.stringLiteralToTypeMap);
                }
            }
            if (g.implicitLexer != null)
            {
                g.ImportVocab(g.implicitLexer);
            }
            //		System.out.println("tokens="+g.tokenNameToTypeMap);
            //		System.out.println("strings="+g.stringLiteralToTypeMap);
            ProcessNonCombinedGrammar(g, gencode);
        }
예제 #7
0
        public virtual void CheckForModeConflicts(Grammar g)
        {
            if (g.IsLexer())
            {
                LexerGrammar lexerGrammar = (LexerGrammar)g;
                foreach (string modeName in lexerGrammar.modes.Keys)
                {
                    if (!modeName.Equals("DEFAULT_MODE") && reservedNames.Contains(modeName))
                    {
                        Rule rule = lexerGrammar.modes[modeName].First();
                        g.tool.errMgr.GrammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, ((CommonTree)rule.ast.Parent).Token, modeName);
                    }

                    if (g.GetTokenType(modeName) != TokenConstants.InvalidType)
                    {
                        Rule rule = lexerGrammar.modes[modeName].First();
                        g.tool.errMgr.GrammarError(ErrorType.MODE_CONFLICTS_WITH_TOKEN, g.fileName, ((CommonTree)rule.ast.Parent).Token, modeName);
                    }
                }
            }
        }
예제 #8
0
        public void testErrors(string[] pairs, bool printTree)
        {
            for (int i = 0; i < pairs.length; i += 2)
            {
                string     input  = pairs[i];
                string     expect = pairs[i + 1];
                ErrorQueue equeue = new ErrorQueue();
                Grammar    g      = null;
                try
                {
                    string[] lines    = input.split("\n");
                    string   fileName = getFilenameFromFirstLineOfGrammar(lines[0]);
                    if (input.startsWith("lexer "))
                    {
                        g = new LexerGrammar(fileName, input, equeue);
                    }
                    else
                    {
                        g = new Grammar(fileName, input, equeue);
                    }
                }
                catch (UnsupportedOperationException ex)
                {
                }
                catch (org.antlr.runtime.RecognitionException re)
                {
                    re.printStackTrace(System.err);
                }
                string actual = equeue.tostring(g != null ? g.tool : new Tool());
                Console.Error.WriteLine(actual);
                string msg = input;
                msg = msg.replaceAll("\n", "\\\\n");
                msg = msg.replaceAll("\r", "\\\\r");
                msg = msg.replaceAll("\t", "\\\\t");

                assertEquals("error in: " + msg, expect, actual);
            }
        }
예제 #9
0
        /**
         * Assign constant values to custom channels defined in a grammar.
         *
         * @param g The grammar.
         * @param channelDefs A collection of AST nodes defining individual channels
         * within a {@code channels{}} block in the grammar.
         */
        internal virtual void AssignChannelTypes(Grammar g, IList <GrammarAST> channelDefs)
        {
            Grammar outermost = g.GetOutermostGrammar();

            foreach (GrammarAST channel in channelDefs)
            {
                string channelName = channel.Text;

                // Channel names can't alias tokens or modes, because constant
                // values are also assigned to them and the ->channel(NAME) lexer
                // command does not distinguish between the various ways a constant
                // can be declared. This method does not verify that channels do not
                // alias rules, because rule names are not associated with constant
                // values in ANTLR grammar semantics.

                if (g.GetTokenType(channelName) != TokenConstants.InvalidType)
                {
                    g.tool.errMgr.GrammarError(ErrorType.CHANNEL_CONFLICTS_WITH_TOKEN, g.fileName, channel.Token, channelName);
                }

                if (LexerATNFactory.COMMON_CONSTANTS.ContainsKey(channelName))
                {
                    g.tool.errMgr.GrammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, channel.Token, channelName);
                }

                if (outermost is LexerGrammar)
                {
                    LexerGrammar lexerGrammar = (LexerGrammar)outermost;
                    if (lexerGrammar.modes.ContainsKey(channelName))
                    {
                        g.tool.errMgr.GrammarError(ErrorType.CHANNEL_CONFLICTS_WITH_MODE, g.fileName, channel.Token, channelName);
                    }
                }

                outermost.DefineChannelName(channel.Text);
            }
        }
예제 #10
0
        /// <summary>
        /// Builds SPICE lexer grammar.
        /// </summary>
        private void BuildGrammar()
        {
            var builder = new LexerGrammarBuilder <SpiceLexerState>();

            builder.AddRegexRule(new LexerInternalRule("LETTER", "[a-zA-Zµ]"));
            builder.AddRegexRule(new LexerInternalRule("CHARACTER", @"[a-zA-Z0-9\-\+§µ]"));
            builder.AddRegexRule(new LexerInternalRule("DIGIT", "[0-9]"));
            builder.AddRegexRule(new LexerInternalRule("SPECIAL", @"[\/\\_\.:%!\#\-;\<\>\^\*\[\]]"));
            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.WHITESPACE,
                                     "A whitespace characters that will be ignored",
                                     @"[\t 	]+",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken,
                                     topRule: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.CONTINUATION_CURRENT_LINE,
                                     "A current line continuation character that is ignored",
                                     @"(\\\\)(\r\n|\r|\n)",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken,
                                     topRule: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.CONTINUATION_NEXT_LINE,
                                     "A next line continuation character that is ignored",
                                     @"((\r\n|\r|\n)[\t 	]*)+\+",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken,
                                     useDecisionProvider: (SpiceLexerState state, string lexem) =>
            {
                if (state.LineNumber == 1 && _options.HasTitle)
                {
                    return(LexerRuleUseDecision.Next);
                }

                return(LexerRuleUseDecision.Use);
            },
                                     topRule: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.TITLE,
                                     "The title - first line of SPICE token",
                                     @"[^\r\n]+",
                                     null,
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.LineNumber == 1 && _options.HasTitle)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            }));

            builder.AddDynamicRule(new LexerDynamicRule(
                                       (int)SpiceTokenType.EXPRESSION_BRACKET,
                                       "A mathematical (also nested) expression in brackets",
                                       @"\{.+",
                                       (string textToLex, LexerState state) =>
            {
                int openBracketCount = 1;
                int i = 0;
                for (i = 1; i < textToLex.Length && openBracketCount > 0; i++)
                {
                    if (textToLex[i] == '}')
                    {
                        openBracketCount--;
                    }

                    if (textToLex[i] == '{')
                    {
                        openBracketCount++;
                    }
                }

                if (openBracketCount == 0)
                {
                    // TODO this is a hack, please refactor me
                    var text     = textToLex.Substring(0, i);
                    var replaced = Regex.Replace(text, @"((\r\n|\r|\n)[\t 	]*)+\+", string.Empty);
                    return(new Tuple <string, int>(replaced, text.Length));
                }

                throw new LexerException("Not matched brackets for expression", new SpiceLineInfo()
                {
                    LineNumber       = state?.LineNumber ?? 0,
                    StartColumnIndex = state?.StartColumnIndex ?? 0,
                });
            }));

            builder.AddDynamicRule(new LexerDynamicRule(
                                       (int)SpiceTokenType.BOOLEAN_EXPRESSION,
                                       "A mathematical (also nested) expression in brackets",
                                       @"\(.+",
                                       (string textToLex, LexerState state) =>
            {
                int openBracketCount = 1;
                var i = 0;
                for (i = 1; i < textToLex.Length && openBracketCount > 0; i++)
                {
                    if (textToLex[i] == ')')
                    {
                        openBracketCount--;
                    }

                    if (textToLex[i] == '(')
                    {
                        openBracketCount++;
                    }
                }

                if (openBracketCount == 0)
                {
                    var text     = textToLex.Substring(0, i);
                    var replaced = Regex.Replace(text, @"((\r\n|\r|\n)[\t 	]*)+\+", string.Empty);
                    return(new Tuple <string, int>(replaced, text.Length));
                }

                throw new LexerException("Not matched brackets for expression", new SpiceLineInfo()
                {
                    LineNumber       = state?.LineNumber ?? 0,
                    StartColumnIndex = state?.StartColumnIndex ?? 0,
                });
            }, new int[] { (int)SpiceTokenType.IF, (int)SpiceTokenType.ELSE_IF }));

            builder.AddDynamicRule(new LexerDynamicRule(
                                       (int)SpiceTokenType.EXPRESSION_BRACKET,
                                       "An expression after equal",
                                       "[^{']+",
                                       (string textToLex, LexerState state) =>
            {
                try
                {
                    var parser = new Parser();
                    var lexer  = new Lexer(textToLex);

                    var node = parser.Parse(lexer);

                    int length = lexer.Index - lexer.BuilderLength;

                    if (lexer.Current == ' ')
                    {
                        length--;
                    }

                    var expression = textToLex.Substring(0, length);

                    return(new Tuple <string, int>(expression, length));
                }
                catch
                {
                    return(new Tuple <string, int>(string.Empty, 0));
                }
            }, new int[] { (int)SpiceTokenType.EQUAL }));

            if (_options.EnableBusSyntax)
            {
                builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                         (int)SpiceTokenType.SUFFIX,
                                         "Suffix notation",
                                         @"[a-zA-Z0-9_§µ]+(<[\d,():*\s]+>)+",
                                         (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.ReturnToken,
                                         (SpiceLexerState state, string lexem) =>
                {
                    return(LexerRuleUseDecision.Use);
                }));

                builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                         (int)SpiceTokenType.PREFIX_SINGLE,
                                         "Prefix notation",
                                         @"\<\*\d\>\s*[a-zA-Z0-9\-\+§µ_]+",
                                         (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.ReturnToken,
                                         (SpiceLexerState state, string lexem) =>
                {
                    return(LexerRuleUseDecision.Use);
                }));

                builder.AddDynamicRule(new LexerDynamicRule(
                                           (int)SpiceTokenType.PREFIX_COMPLEX,
                                           "Prefix notation with brackets",
                                           @"\<\*\d\>\s*\(",
                                           (string textToLex, LexerState state) =>
                {
                    Match tokenMatch = state.CurrentRuleRegex.Match(textToLex, 0, textToLex.Length);

                    if (tokenMatch.Success && tokenMatch.Length > 0)
                    {
                        string prefixBeginning = tokenMatch.Value;

                        int openBracketCount = 1;
                        var i = 0;
                        for (i = prefixBeginning.Length; i < textToLex.Length && openBracketCount > 0; i++)
                        {
                            if (textToLex[i] == ')')
                            {
                                openBracketCount--;
                            }

                            if (textToLex[i] == '(')
                            {
                                openBracketCount++;
                            }
                        }

                        if (openBracketCount == 0)
                        {
                            var complexPrefix = textToLex.Substring(0, i);
                            return(new Tuple <string, int>(complexPrefix, complexPrefix.Length));
                        }
                    }

                    throw new LexerException("Error with prefix complex notatation", new SpiceLineInfo()
                    {
                        LineNumber       = state?.LineNumber ?? 0,
                        StartColumnIndex = state?.StartColumnIndex ?? 0,
                    });
                }));
            }

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.DOT,
                    "A dot character",
                    "\\."));

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.COMMA,
                    "A comma character",
                    ","));

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.DELIMITER,
                    "A delimiter character",
                    @"(\(|\)|\|)",
                    (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.ReturnToken,
                    (SpiceLexerState state, string lexem) =>
            {
                if (state.PreviousReturnedTokenType == (int)SpiceTokenType.IF || (state.PreviousReturnedTokenType == (int)SpiceTokenType.ELSE_IF))
                {
                    return(LexerRuleUseDecision.Next);
                }

                return(LexerRuleUseDecision.Use);
            }));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COM_START,
                                     "An block comment start",
                                     @"#COM",
                                     (SpiceLexerState state, string lexem) =>
            {
                state.InCommentBlock = true;
                return(LexerRuleReturnDecision.IgnoreToken);
            },
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.NewLine || state.PreviousReturnedTokenType == 0)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            },
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COM_END,
                                     "An block comment end",
                                     "#ENDCOM",
                                     (SpiceLexerState state, string lexem) =>
            {
                state.InCommentBlock = false;
                return(LexerRuleReturnDecision.IgnoreToken);
            },
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.InCommentBlock)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            },
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COM_CONTENT,
                                     "An block comment content",
                                     @".*",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken,
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.InCommentBlock)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            }));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.EQUAL,
                                     "An equal character",
                                     @"="));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.NEWLINE,
                                     "A new line characters",
                                     @"(\r\n|\n|\r)",
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.InCommentBlock)
                {
                    return(LexerRuleReturnDecision.IgnoreToken);
                }

                return(LexerRuleReturnDecision.ReturnToken);
            }));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ENDS,
                                     ".ENDS keyword",
                                     "\\.ENDS",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.END,
                                     ".END keyword",
                                     "\\.END",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ENDL,
                                     ".ENDL keyword",
                                     "\\.ENDL",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.IF,
                                     ".IF keyword",
                                     "\\.IF",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ENDIF,
                                     ".ENDIF keyword",
                                     "\\.ENDIF",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ELSE,
                                     ".ELSE keyword",
                                     "\\.ELSE",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ELSE_IF,
                                     ".ELSEIF keyword",
                                     "\\.ELSEIF",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.MODEL,
                                     ".MODEL keyword",
                                     "\\.MODEL",
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.VALUE,
                                     "A value with dot separator",
                                     @"([+-]?((<DIGIT>)+(\.(<DIGIT>)*)?|\.(<DIGIT>)+)(e(\+|-)?(<DIGIT>)+)?[tgmkunpf]?(<LETTER>)*)",
                                     null,
                                     (SpiceLexerState state, string lexem) => LexerRuleUseDecision.Use,
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.PERCENT,
                                     "A percent value with dot separator",
                                     @"([+-]?((<DIGIT>)+(\.(<DIGIT>)*)?|\.(<DIGIT>)+)(e(\+|-)?(<DIGIT>)+)?[tgmkunpf]?(<LETTER>)*)%",
                                     null,
                                     (SpiceLexerState state, string lexem) => LexerRuleUseDecision.Use,
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COMMENT_HSPICE,
                                     "A comment - HSpice style",
                                     @"\$[^\r\n]*",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COMMENT_PSPICE,
                                     "A comment - PSpice style",
                                     @";[^\r\n]*",
                                     (SpiceLexerState state, string lexem) => LexerRuleReturnDecision.IgnoreToken));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.COMMENT,
                                     "A full line comment",
                                     @"\*[^\r\n]*",
                                     null,
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.LineNumber == 1 && _options.HasTitle)
                {
                    return(LexerRuleUseDecision.Next);
                }

                if (state.NewLine)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            },
                                     ignoreCase: true));

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.DOUBLE_QUOTED_STRING,
                    "A string with double quotation marks",
                    "\"(?:[^\"\\\\]|\\\\.)*\""));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.EXPRESSION_SINGLE_QUOTES,
                                     "A mathematical expression in single quotes",
                                     @"'[^']*'",
                                     null,
                                     (SpiceLexerState state, string lexem) =>
            {
                if (state.PreviousReturnedTokenType == (int)SpiceTokenType.EQUAL)
                {
                    return(LexerRuleUseDecision.Use);
                }

                return(LexerRuleUseDecision.Next);
            },
                                     ignoreCase: !_options.IsDotStatementNameCaseSensitive));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.EXPRESSION_BRACKET,
                                     "A mathematical expression in brackets",
                                     @"{[^{}\r\n]*}",
                                     ignoreCase: true));

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.SINGLE_QUOTED_STRING,
                    "A string with single quotation marks",
                    @"'[^']*'"));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.REFERENCE,
                                     "A reference",
                                     "@(<CHARACTER>(<CHARACTER>|<SPECIAL>)*)",
                                     null,
                                     (SpiceLexerState state, string lexem) =>
            {
                if (lexem.EndsWith("\\", StringComparison.Ordinal) && state.BeforeLineBreak)
                {
                    return(LexerRuleUseDecision.Next);
                }

                return(LexerRuleUseDecision.Use);
            },
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.WORD,
                                     "A word",
                                     "<LETTER>(<CHARACTER>|<SPECIAL>)*",
                                     null,
                                     (SpiceLexerState state, string lexem) => LexerRuleUseDecision.Use,
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.WORD,
                                     "A relative path",
                                     @"(\.\.\\|\.\.\/|\.\/|\.\\)(<CHARACTER>|<SPECIAL>)*",
                                     null,
                                     (SpiceLexerState state, string lexem) => LexerRuleUseDecision.Use,
                                     ignoreCase: true));

            builder.AddRegexRule(
                new LexerTokenRule <SpiceLexerState>(
                    (int)SpiceTokenType.IDENTIFIER,
                    "An identifier",
                    @"((<CHARACTER>|_|\*)(<CHARACTER>|<SPECIAL>)*)",
                    null,
                    (SpiceLexerState state, string lexem) => LexerRuleUseDecision.Use,
                    ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.REFERENCE,
                                     "A reference",
                                     "@(<CHARACTER>(<CHARACTER>|<SPECIAL>)+)",
                                     ignoreCase: true));

            builder.AddRegexRule(new LexerTokenRule <SpiceLexerState>(
                                     (int)SpiceTokenType.ASTERIKS,
                                     "An asterisk character",
                                     "\\*"));

            _grammar = builder.GetGrammar();
        }
예제 #11
0
 public static IDictionary <Rule, ISet <Rule> > GetRuleDependencies(LexerGrammar g, string modeName)
 {
     return(GetRuleDependencies(g, g.modes[modeName]));
 }