public void SkipTokenTest()
{
Lexicon lexicon = new Lexicon();
LexerState global = lexicon.DefaultLexer;
LexerState keywords = global.DefineSubState();
LexerState xml = keywords.DefineSubState();
var ID = global.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = global.DefineToken(RE.Range('0', '9').Many1());
var WHITESPACE = global.DefineToken(RE.Symbol(' ').Many());
var ERROR = global.DefineToken(RE.Range(Char.MinValue, (char)255));
var IF = keywords.DefineToken(RE.Literal("if"));
var ELSE = keywords.DefineToken(RE.Literal("else"));
var XMLNS = xml.DefineToken(RE.Literal("xmlns"));
ScannerInfo info = lexicon.CreateScannerInfo();
PeekableScanner scanner = new PeekableScanner(info);
string source = "asdf04a 1107 else Z if vvv xmlns 772737";
StringReader sr = new StringReader(source);
scanner.SetSource(new SourceReader(sr));
scanner.SetSkipTokens(WHITESPACE.Index, ERROR.Index);
info.LexerStateIndex = xml.Index;
Lexeme l1 = scanner.Read();
Assert.AreEqual(ID.Index, l1.TokenIndex);
Assert.AreEqual("asdf04a", l1.Value);
Assert.AreEqual(0, l1.SkippedTokenCount);
Lexeme l2 = scanner.Read();
Assert.AreEqual(NUM.Index, l2.TokenIndex);
Assert.AreEqual("1107", l2.Value);
Assert.AreEqual(1, l2.SkippedTokenCount);
Lexeme l3 = scanner.Read();
Assert.AreEqual(ELSE.Index, l3.TokenIndex);
Assert.AreEqual("else", l3.Value);
Assert.AreEqual(1, l2.SkippedTokenCount);
Lexeme l4 = scanner.Read();
Assert.AreEqual(IF.Index, l4.TokenIndex);
Assert.AreEqual("if", l4.Value);
Assert.AreEqual(3, l4.SkippedTokenCount);
int p1 = scanner.Peek();
Assert.AreEqual(ID.Index, p1);
int p2 = scanner.Peek2();
int p3 = scanner.Peek(3);
int peof = scanner.Peek(4);
Assert.AreEqual(info.EndOfStreamTokenIndex, peof);
Lexeme l6 = scanner.Read();
Lexeme l7 = scanner.Read();
Assert.AreEqual(XMLNS.Index, l7.TokenIndex);
Lexeme l8 = scanner.Read();
Assert.AreEqual(NUM.Index, l8.TokenIndex);
Lexeme leof = scanner.Read();
Assert.AreEqual(info.EndOfStreamTokenIndex, leof.TokenIndex);
Assert.AreEqual(leof.Span.StartLocation.CharIndex, leof.Span.EndLocation.CharIndex);
Assert.AreEqual(source.Length, leof.Span.StartLocation.CharIndex);
}
public void ErrorRecoveryTest()
{
Lexicon lexicon = new Lexicon();
LexerState global = lexicon.DefaultLexer;
var ID = global.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = global.DefineToken(RE.Range('0', '9').Many1());
var WHITESPACE = global.DefineToken(RE.Symbol(' ').Many());
ScannerInfo info = lexicon.CreateScannerInfo();
PeekableScanner scanner = new PeekableScanner(info);
string source = "asdf04a 1107 !@#$!@ Z if vvv xmlns 772737";
StringReader sr = new StringReader(source);
scanner.SetSource(new SourceReader(sr));
scanner.SetSkipTokens(WHITESPACE.Index);
scanner.RecoverErrors = true;
CompilationErrorManager em = new CompilationErrorManager();
em.DefineError(101, 0, CompilationStage.Scanning, "Invalid token: {0}");
scanner.ErrorManager = em;
scanner.LexicalErrorId = 101;
Lexeme l1 = scanner.Read();
Assert.AreEqual(ID.Index, l1.TokenIndex);
Lexeme l2 = scanner.Read();
Assert.AreEqual(NUM.Index, l2.TokenIndex);
Assert.AreEqual(0, em.Errors.Count);
Lexeme l3 = scanner.Read();
Assert.AreEqual(ID.Index, l3.TokenIndex);
Assert.IsTrue(em.Errors.Count > 0);
Assert.AreEqual(101, em.Errors[0].Info.Id);
}
