private void OnInitialize()
{
m_lexicon = new Lexicon();
OnDefineLexer(m_lexicon, m_triviaTokens);
m_scannerInfo = OnCreateScannerInfo();
var parser = OnDefineParser();
if (parser == null)
{
throw new InvalidOperationException("Parser not defined");
}
m_parser = parser.SuffixedBy(Parsers.Eos());
m_context = new ParserContext(m_errorManager, m_unexpectedTokenErrorId, m_missingTokenErrorId);
OnDefineParserErrors(m_errorManager);
m_parserRunner = new ParserRunner <T>(m_parser, m_context);
m_scannerBuilder = new ForkableScannerBuilder(m_scannerInfo);
m_scannerBuilder.SetTriviaTokens(m_triviaTokens.Select(t => t.Index).ToArray());
m_scannerBuilder.RecoverErrors = true;
m_scannerBuilder.LexicalErrorId = m_lexicalErrorId;
m_isInitialized = true;
}
public void Test(SourceReader sr)
{
Console.WriteLine("=============== CPS Parser Combinators ==================");
SetUpScanner();
var production = SetUpParser();
ForkableScannerBuilder fsb = new ForkableScannerBuilder(m_scannerInfo);
fsb.SetTriviaTokens(SPACE.Index);
var scanner = fsb.Create(sr);
var runner = new ParserRunner <int>(production);
try
{
var result = runner.Execute(scanner);
Console.WriteLine("Result: {0}", result);
}
catch (Exception ex)
{
Console.WriteLine("Parse Errors:");
Console.WriteLine(ex.Message);
}
Console.WriteLine();
Console.WriteLine();
}
public void ParserFuncTest()
{
Lexicon test = new Lexicon();
var ID = test.DefaultLexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.DefaultLexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.DefaultLexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.DefaultLexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER.AsParser(l => l.PrefixTrivia.Count == 0)
from n in NUM
select "A";
var p2 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "B";
var parser1 = p1 | p2;
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetTriviaTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
ParserRunner <string> runner = new ParserRunner <string>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("A", result1);
Assert.AreEqual(0, errorManager.Errors.Count);
string source2 = "abc > > 123";
var sr2 = new SourceReader(new StringReader(source2));
ForkableScanner scanner2 = builder.Create(sr2);
var result2 = runner.Run(scanner2);
Assert.AreEqual("B", result2);
Assert.AreEqual(0, errorManager.Errors.Count);
}
public void ForkableScannerTest()
{
Lexicon lexicon = new Lexicon();
var A = lexicon.DefaultLexer.DefineToken(RE.Range('a', 'z'));
ScannerInfo si = lexicon.CreateScannerInfo();
string source = "abcdefghijklmnopqrstuvwxyz";
ForkableScannerBuilder fsBuilder = new ForkableScannerBuilder(si);
ForkableScanner fscanner = fsBuilder.Create(new SourceReader(new StringReader(source)));
var l1 = fscanner.Read();
Assert.AreEqual("a", l1.Value);
var l2 = fscanner.Read();
Assert.AreEqual("b", l2.Value);
//fork
ForkableScanner fscanner2 = fscanner.Fork();
for (int i = 2; i <= 4; i++)
{
var l = fscanner.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
for (int i = 2; i <= 5; i++)
{
var l = fscanner2.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
ForkableScanner fscanner3 = fscanner.Fork();
var l5a = fscanner.Read();
var l5b = fscanner3.Read();
Assert.AreEqual(source[5].ToString(), l5a.Value);
Assert.AreEqual(source[5].ToString(), l5b.Value);
var l6b = fscanner2.Read();
var l6a = fscanner3.Read();
Assert.AreEqual(source[6].ToString(), l6a.Value);
Assert.AreEqual(source[6].ToString(), l6b.Value);
var l7a = fscanner2.Read();
for (int i = 7; i < 9; i++)
{
var l = fscanner3.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
}
public void ParserCastTest()
{
Lexicon test = new Lexicon();
var ID = test.Lexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.Lexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.Lexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.Lexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "hello";
var parser1 = p1.TryCast <object>();
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetTriviaTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
var el = errorManager.CreateErrorList();
context.ErrorList = el;
ParserRunner <object> runner = new ParserRunner <object>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("hello", result1);
Assert.AreEqual(0, el.Count);
}
public void Test(SourceReader sr)
{
Console.WriteLine("=============== Error Recovery Parser Combinators =======");
SetUpScanner();
var production = SetUpParser();
ForkableScannerBuilder fsb = new ForkableScannerBuilder(m_scannerInfo);
fsb.SetTriviaTokens(SPACE.Index);
var scanner = fsb.Create(sr);
var runner = new ParserRunner <int>(production);
List <SyntaxError> errors = new List <SyntaxError>();
var result = runner.Execute(scanner, errors);
if (errors.Count == 0)
{
Console.WriteLine("Result: {0}", result);
}
else
{
Console.WriteLine("Parse Errors:");
foreach (var err in errors)
{
Console.WriteLine("{0}:{1} {2}",
err.Code,
err.Description,
err.Location.StartLocation);
}
}
Console.WriteLine();
Console.WriteLine();
}
public void ParserErrorRecoveryTest()
{
Lexicon binaryTreeSyntax = new Lexicon();
var lex = binaryTreeSyntax.Lexer;
//lex
Token LEFTPH = lex.DefineToken(RE.Symbol('('));
Token RIGHTPH = lex.DefineToken(RE.Symbol(')'));
Token COMMA = lex.DefineToken(RE.Symbol(','));
Token LETTER = lex.DefineToken(RE.Range('a', 'z') | RE.Range('A', 'Z'), "ID");
//grammar
Production <Node> NodeParser = new Production <Node>();
NodeParser.Rule =
(from a in LETTER
from _1 in LEFTPH
from left in NodeParser
from _2 in COMMA
from right in NodeParser
from _3 in RIGHTPH
select new Node(a.Value.Content, left, right))
| Grammar.Empty <Node>(null);
var builder = new ForkableScannerBuilder(binaryTreeSyntax.CreateScannerInfo());
const string correct = "A(B(,),C(,))";
string source = "A((B(,),C(,)";
SourceReader sr = new SourceReader(
new StringReader(source));
var info = binaryTreeSyntax.CreateScannerInfo();
Scanner scanner = new Scanner(info);
scanner.SetSource(sr);
CompilationErrorManager errorManager = new CompilationErrorManager();
errorManager.DefineError(1, 0, CompilationStage.Parsing, "Unexpected token '{0}'");
errorManager.DefineError(2, 0, CompilationStage.Parsing, "Missing token '{0}'");
errorManager.DefineError(3, 0, CompilationStage.Parsing, "Invalid token found, did you mean '{0}' ?");
errorManager.DefineError(4, 0, CompilationStage.Parsing, "Syntax error");
ProductionInfoManager pim = new ProductionInfoManager(NodeParser.SuffixedBy(Grammar.Eos()));
LR0Model lr0 = new LR0Model(pim);
lr0.BuildModel();
string dot = lr0.ToString();
TransitionTable tt = TransitionTable.Create(lr0, info);
SyntaxErrors errDef = new SyntaxErrors()
{
TokenUnexpectedId = 1, TokenMissingId = 2, OtherErrorId = 4, TokenMistakeId = 3
};
ParserEngine driver = new ParserEngine(tt, errDef);
Lexeme r;
do
{
r = scanner.Read();
driver.Input(r);
} while (!r.IsEndOfStream);
var result = driver.GetResult(0, errorManager.CreateErrorList());
;
}
public void ParserDriverSimpleTest()
{
Lexicon test = new Lexicon();
var X = test.Lexer.DefineToken(RE.Symbol('x'));
var PLUS = test.Lexer.DefineToken(RE.Symbol('+'));
var scannerinfo = test.CreateScannerInfo();
Production <object> E = new Production <object>(), T = new Production <object>();
E.Rule =
(from t in T
from plus in PLUS
from e in E
select(object)(((int)t) + ((int)e))) | T;
T.Rule =
from x in X
select(object) 1;
ProductionInfoManager pim = new ProductionInfoManager(E.SuffixedBy(Grammar.Eos()));
LR0Model lr0 = new LR0Model(pim);
lr0.BuildModel();
string dot = lr0.ToString();
TransitionTable tt = TransitionTable.Create(lr0, scannerinfo);
ParserEngine driver = new ParserEngine(tt, new SyntaxErrors()
{
TokenUnexpectedId = 1
});
ForkableScannerBuilder builder = new ForkableScannerBuilder(scannerinfo);
var em = new CompilationErrorManager();;
var el = em.CreateErrorList();
builder.ErrorList = el;
var scanner = builder.Create(new SourceReader(new StringReader("x+x+x")));
var z1 = scanner.Read();
driver.Input(z1);
var z2 = scanner.Read();
driver.Input(z2);
var z3 = scanner.Read();
driver.Input(z3);
var z4 = scanner.Read();
driver.Input(z4);
var z5 = scanner.Read();
driver.Input(z5);
var z6 = scanner.Read();
driver.Input(z6);
Assert.AreEqual(0, driver.CurrentStackCount);
Assert.AreEqual(1, driver.AcceptedCount);
Assert.AreEqual(3, driver.GetResult(0, null));
}
public void ParserDriverConflictTest()
{
Lexicon test = new Lexicon();
var X = test.Lexer.DefineToken(RE.Symbol('x'));
var PLUS = test.Lexer.DefineToken(RE.Symbol('+'));
var ASTERISK = test.Lexer.DefineToken(RE.Symbol('*'));
var scannerinfo = test.CreateScannerInfo();
Production <object> E = new Production <object>(), T = new Production <object>();
E.Rule =
(from e1 in E
from plus in PLUS
from e2 in E
select(object)(((int)e1) + ((int)e2))) |
(from e1 in E
from mul in ASTERISK
from e2 in E
select(object)(((int)e1) * ((int)e2))) | T;
T.Rule =
from x in X
select(object) 2;
ProductionInfoManager pim = new ProductionInfoManager(E.SuffixedBy(Grammar.Eos()));
LR0Model lr0 = new LR0Model(pim);
lr0.BuildModel();
string dot = lr0.ToString();
TransitionTable tt = TransitionTable.Create(lr0, scannerinfo);
ParserEngine driver = new ParserEngine(tt, new SyntaxErrors());
ForkableScannerBuilder builder = new ForkableScannerBuilder(scannerinfo);
var em = new CompilationErrorManager();
var el = em.CreateErrorList();
builder.ErrorList = el;
var scanner = builder.Create(new SourceReader(new StringReader("x+x*x")));
var z1 = scanner.Read();
driver.Input(z1);
var z2 = scanner.Read();
driver.Input(z2);
var z3 = scanner.Read();
driver.Input(z3);
var z4 = scanner.Read();
driver.Input(z4);
var z5 = scanner.Read();
driver.Input(z5);
var z6 = scanner.Read();
driver.Input(z6);
Assert.AreEqual(0, driver.CurrentStackCount);
Assert.AreEqual(2, driver.AcceptedCount);
var results = new[] { (int)driver.GetResult(0, null), (int)driver.GetResult(1, null) };
Assert.IsTrue(results.Contains(8));
Assert.IsTrue(results.Contains(6));
}