Example #1
0
        public ForkableScanner Create(SourceReader source)
        {
            CodeContract.RequiresArgumentNotNull(source, "source");

            Scanner masterScanner = new Scanner(ScannerInfo);
            masterScanner.SetSource(source);
            masterScanner.SetSkipTokens(m_skipTokens);
            masterScanner.ErrorManager = ErrorManager;
            masterScanner.RecoverErrors = RecoverErrors;
            masterScanner.LexicalErrorId = LexicalErrorId;

            return ForkableScanner.Create(masterScanner);
        }
Example #2
0
        public void CompactCharSetTest()
        {
            Lexicon lexicon = new Lexicon();
            LexerState global = lexicon.DefaultLexer;
            LexerState keywords = global.DefineSubState();
            LexerState xml = keywords.DefineSubState();

            var lettersCategories = new[] { UnicodeCategory.LetterNumber,
                                            UnicodeCategory.LowercaseLetter,
                                            UnicodeCategory.ModifierLetter,
                                            UnicodeCategory.OtherLetter,
                                            UnicodeCategory.TitlecaseLetter,
                                            UnicodeCategory.UppercaseLetter};

            var RE_IDCHAR = RE.CharsOf(c => lettersCategories.Contains(Char.GetUnicodeCategory(c)));

            var ID = global.DefineToken(RE_IDCHAR.Concat(
                (RE_IDCHAR | RE.Range('0', '9')).Many()));
            var NUM = global.DefineToken(RE.Range('0', '9').Many1());
            var WHITESPACE = global.DefineToken(RE.Symbol(' ').Many());

            var IF = keywords.DefineToken(RE.Literal("if"));
            var ELSE = keywords.DefineToken(RE.Literal("else"));

            var XMLNS = xml.DefineToken(RE.Literal("xmlns"));

            var scannerInfo = lexicon.CreateScannerInfo();
            scannerInfo.LexerStateIndex = xml.Index;

            Scanner s = new Scanner(scannerInfo);

            string source = "xmlns 你好吗1 123 蘏臦囧綗 ABCD if";

            SourceReader sr = new SourceReader(new StringReader(source));

            s.SetSource(sr);
            s.SetSkipTokens(WHITESPACE.Index);

            var l1 = s.Read();
            Assert.AreEqual(XMLNS.Index, l1.TokenIndex);

            var l2 = s.Read();
            Assert.AreEqual(ID.Index, l2.TokenIndex);

            var l3 = s.Read();
            Assert.AreEqual(NUM.Index, l3.TokenIndex);

            var l4 = s.Read();
            Assert.AreEqual(ID.Index, l4.TokenIndex);

            var l5 = s.Read();
            Assert.AreEqual(ID.Index, l5.TokenIndex);

            var l6 = s.Read();
            Assert.AreEqual(IF.Index, l6.TokenIndex);
        }