Пример #1
0
        private void InitializeStaticLexer(IdentifierType idType = IdentifierType.Alpha,
                                           params GenericToken[] staticTokens)
        {
            FSMBuilder    = new FSMLexerBuilder <GenericToken>();
            StringCounter = 0;

            // conf
            FSMBuilder.IgnoreWS()
            .WhiteSpace(' ')
            .WhiteSpace('\t')
            .IgnoreEOL();

            // start machine definition
            FSMBuilder.Mark(start);

            if (staticTokens.ToList().Contains(GenericToken.Identifier) ||
                staticTokens.ToList().Contains(GenericToken.KeyWord))
            {
                InitializeIdentifier(idType);
            }

            //numeric
            if (staticTokens.ToList().Contains(GenericToken.Int) || staticTokens.ToList().Contains(GenericToken.Double))
            {
                FSMBuilder = FSMBuilder.GoTo(start)
                             .RangeTransition('0', '9')
                             .Mark(in_int)
                             .RangeTransitionTo('0', '9', in_int)
                             .End(GenericToken.Int);
                if (staticTokens.ToList().Contains(GenericToken.Double))
                {
                    FSMBuilder.Transition('.')
                    .Mark(start_double)
                    .RangeTransition('0', '9')
                    .Mark(in_double)
                    .RangeTransitionTo('0', '9', in_double)
                    .End(GenericToken.Double);
                }
            }

            LexerFsm = FSMBuilder.Fsm;
        }
Пример #2
0
        private void InitializeStaticLexer(Config config, GenericToken[] staticTokens)
        {
            FSMBuilder    = new FSMLexerBuilder <GenericToken>();
            StringCounter = 0;

            // conf
            FSMBuilder
            .IgnoreWS(config.IgnoreWS)
            .WhiteSpace(config.WhiteSpace)
            .IgnoreEOL(config.IgnoreEOL);

            // start machine definition
            FSMBuilder.Mark(start);

            if (staticTokens.Contains(GenericToken.Identifier) || staticTokens.Contains(GenericToken.KeyWord))
            {
                InitializeIdentifier(config);
            }

            // numeric
            if (staticTokens.Contains(GenericToken.Int) || staticTokens.Contains(GenericToken.Double))
            {
                FSMBuilder = FSMBuilder.GoTo(start)
                             .RangeTransition('0', '9')
                             .Mark(in_int)
                             .RangeTransitionTo('0', '9', in_int)
                             .End(GenericToken.Int);
                if (staticTokens.Contains(GenericToken.Double))
                {
                    FSMBuilder.Transition('.')
                    .Mark(start_double)
                    .RangeTransition('0', '9')
                    .Mark(in_double)
                    .RangeTransitionTo('0', '9', in_double)
                    .End(GenericToken.Double);
                }
            }

            LexerFsm = FSMBuilder.Fsm;
        }
Пример #3
0
        private static void testLexerBuilder()
        {
            var builder = new FSMLexerBuilder <JsonToken>();


            // conf
            builder.IgnoreWS()
            .WhiteSpace(' ')
            .WhiteSpace('\t')
            .IgnoreEOL();

            // start machine definition
            builder.Mark("start");


            // string literal
            builder.Transition('\"')
            .Mark("in_string")
            .ExceptTransitionTo(new[] { '\"', '\\' }, "in_string")
            .Transition('\\')
            .Mark("escape")
            .AnyTransitionTo(' ', "in_string")
            .Transition('\"')
            .End(JsonToken.STRING)
            .Mark("string_end")
            .CallBack(match =>
            {
                match.Result.Value = match.Result.Value.ToUpper();
                return(match);
            });

            // accolades
            builder.GoTo("start")
            .Transition('{')
            .End(JsonToken.ACCG);

            builder.GoTo("start")
            .Transition('}')
            .End(JsonToken.ACCD);

            // corchets
            builder.GoTo("start")
            .Transition('[')
            .End(JsonToken.CROG);

            builder.GoTo("start")
            .Transition(']')
            .End(JsonToken.CROD);

            // 2 points
            builder.GoTo("start")
            .Transition(':')
            .End(JsonToken.COLON);

            // comma
            builder.GoTo("start")
            .Transition(',')
            .End(JsonToken.COMMA);

            //numeric
            builder.GoTo("start")
            .RangeTransition('0', '9')
            .Mark("in_int")
            .RangeTransitionTo('0', '9', "in_int")
            .End(JsonToken.INT)
            .Transition('.')
            .Mark("start_double")
            .RangeTransition('0', '9')
            .Mark("in_double")
            .RangeTransitionTo('0', '9', "in_double")
            .End(JsonToken.DOUBLE);


            var code = "{\n\"d\" : 42.42 ,\n\"i\" : 42 ,\n\"s\" : \"quarante-deux\",\n\"s2\":\"a\\\"b\"\n}";
            //code = File.ReadAllText("test.json");
            var lex   = builder.Fsm;
            var r     = lex.Run(code, 0);
            var total = "";

            while (r.IsSuccess)
            {
                var msg = $"{r.Result.TokenID} : {r.Result.Value} @{r.Result.Position}";
                total += msg + "\n";
                Console.WriteLine(msg);
                r = lex.Run(code);
            }
        }