public ForkableScanner Create(SourceReader source)
{
CodeContract.RequiresArgumentNotNull(source, "source");
Scanner masterScanner = new Scanner(ScannerInfo);
masterScanner.SetSource(source);
masterScanner.SetSkipTokens(m_skipTokens);
masterScanner.ErrorManager = ErrorManager;
masterScanner.RecoverErrors = RecoverErrors;
masterScanner.LexicalErrorId = LexicalErrorId;
return ForkableScanner.Create(masterScanner);
}
public void CompactCharSetTest()
{
Lexicon lexicon = new Lexicon();
LexerState global = lexicon.DefaultLexer;
LexerState keywords = global.DefineSubState();
LexerState xml = keywords.DefineSubState();
var lettersCategories = new[] { UnicodeCategory.LetterNumber,
UnicodeCategory.LowercaseLetter,
UnicodeCategory.ModifierLetter,
UnicodeCategory.OtherLetter,
UnicodeCategory.TitlecaseLetter,
UnicodeCategory.UppercaseLetter};
var RE_IDCHAR = RE.CharsOf(c => lettersCategories.Contains(Char.GetUnicodeCategory(c)));
var ID = global.DefineToken(RE_IDCHAR.Concat(
(RE_IDCHAR | RE.Range('0', '9')).Many()));
var NUM = global.DefineToken(RE.Range('0', '9').Many1());
var WHITESPACE = global.DefineToken(RE.Symbol(' ').Many());
var IF = keywords.DefineToken(RE.Literal("if"));
var ELSE = keywords.DefineToken(RE.Literal("else"));
var XMLNS = xml.DefineToken(RE.Literal("xmlns"));
var scannerInfo = lexicon.CreateScannerInfo();
scannerInfo.LexerStateIndex = xml.Index;
Scanner s = new Scanner(scannerInfo);
string source = "xmlns 你好吗1 123 蘏臦囧綗 AＢＣＤ if";
SourceReader sr = new SourceReader(new StringReader(source));
s.SetSource(sr);
s.SetSkipTokens(WHITESPACE.Index);
var l1 = s.Read();
Assert.AreEqual(XMLNS.Index, l1.TokenIndex);
var l2 = s.Read();
Assert.AreEqual(ID.Index, l2.TokenIndex);
var l3 = s.Read();
Assert.AreEqual(NUM.Index, l3.TokenIndex);
var l4 = s.Read();
Assert.AreEqual(ID.Index, l4.TokenIndex);
var l5 = s.Read();
Assert.AreEqual(ID.Index, l5.TokenIndex);
var l6 = s.Read();
Assert.AreEqual(IF.Index, l6.TokenIndex);
}