public Scanner(ScannerInfo scannerInfo)
{
m_scannerInfo = scannerInfo;
m_engine = new FiniteAutomationEngine(m_scannerInfo.TransitionTable, m_scannerInfo.CharClassTable);
m_lexemeValueBuilder = new StringBuilder(32);
m_tokenAttributes = new int[scannerInfo.TokenCount];
Initialize();
}
public Scanner(ScannerInfo scannerInfo)
{
m_scannerInfo = scannerInfo;
m_engine = new FiniteAutomationEngine(m_scannerInfo.TransitionTable, m_scannerInfo.CharClassTable);
m_lexemeValueBuilder = new StringBuilder(32);
m_tokenAttributes = new int[scannerInfo.TokenCount];
Initialize();
}
public void LexerStateToDFATest()
{
Lexicon lexicon = new Lexicon();
Lexer global = lexicon.Lexer;
Lexer keywords = global.CreateSubLexer();
Lexer xml = keywords.CreateSubLexer();
var ID = global.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = global.DefineToken(RE.Range('0', '9').Many1());
var ERROR = global.DefineToken(RE.Range(Char.MinValue, (char)255));
var IF = keywords.DefineToken(RE.Literal("if"));
var ELSE = keywords.DefineToken(RE.Literal("else"));
var XMLNS = xml.DefineToken(RE.Literal("xmlns"));
DFAModel dfa = DFAModel.Create(lexicon);
CompressedTransitionTable tc = CompressedTransitionTable.Compress(dfa);
ScannerInfo si = lexicon.CreateScannerInfo();
FiniteAutomationEngine engine = new FiniteAutomationEngine(si.TransitionTable, si.CharClassTable);
engine.InputString("if");
Assert.AreEqual(ID.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("12345");
Assert.AreEqual(NUM.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("asdf12dd");
Assert.AreEqual(ID.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("A");
Assert.AreEqual(ERROR.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("AAA");
Assert.IsTrue(engine.IsAtStoppedState);
engine.Reset();
engine.InputString("if ");
Assert.IsTrue(engine.IsAtStoppedState);
engine.Reset();
si.CurrentLexerIndex = keywords.Index;
engine.InputString("if");
Assert.AreEqual(IF.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("else");
Assert.AreEqual(ELSE.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("xmlns");
Assert.AreEqual(ID.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
si.CurrentLexerIndex = xml.Index;
engine.InputString("if");
Assert.AreEqual(IF.Index, si.GetTokenIndex(engine.CurrentState));
engine.Reset();
engine.InputString("xml");
Assert.IsFalse(engine.IsAtStoppedState);
engine.Reset();
engine.InputString("xmlns");
Assert.AreEqual(XMLNS.Index, si.GetTokenIndex(engine.CurrentState));
;
}