static void Main(string[] args)
{
string input = "1 + 2 * 3";
SourceReader sr = new SourceReader(new StringReader(input));
var startPoint = sr.CreateRevertPoint();
Console.WriteLine("Input String: ");
Console.WriteLine(input);
Console.WriteLine();
SimpleCombinatorsTest test1 = new SimpleCombinatorsTest();
test1.Test(sr);
sr.Revert(startPoint);
CpsCombinatorsTest test2 = new CpsCombinatorsTest();
test2.Test(sr);
sr.Revert(startPoint);
ErrorRecoveryCombinatorsTest test3 = new ErrorRecoveryCombinatorsTest();
test3.Test(sr);
sr.Revert(startPoint);
GLRCombinatorsTest test4 = new GLRCombinatorsTest();
test4.Test(sr);
sr.Revert(startPoint);
}
public void ParserFuncTest()
{
Lexicon test = new Lexicon();
var ID = test.DefaultLexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.DefaultLexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.DefaultLexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.DefaultLexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER.AsParser(true)
from n in NUM
select "A";
var p2 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "B";
var parser1 = p1 | p2;
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetSkipTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
ParserRunner<string> runner = new ParserRunner<string>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("A", result1);
Assert.AreEqual(0, errorManager.Errors.Count);
string source2 = "abc > > 123";
var sr2 = new SourceReader(new StringReader(source2));
ForkableScanner scanner2 = builder.Create(sr2);
var result2 = runner.Run(scanner2);
Assert.AreEqual("B", result2);
Assert.AreEqual(0, errorManager.Errors.Count);
}
public void ParserCastTest()
{
Lexicon test = new Lexicon();
var ID = test.Lexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.Lexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.Lexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.Lexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "hello";
var parser1 = p1.TryCast<object>();
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetTriviaTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
var el = errorManager.CreateErrorList();
context.ErrorList = el;
ParserRunner<object> runner = new ParserRunner<object>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("hello", result1);
Assert.AreEqual(0, el.Count);
}
public void CompactCharSetTest()
{
Lexicon lexicon = new Lexicon();
Lexer global = lexicon.Lexer;
Lexer keywords = global.CreateSubLexer();
Lexer xml = keywords.CreateSubLexer();
var lettersCategories = new[] { UnicodeCategory.LetterNumber,
UnicodeCategory.LowercaseLetter,
UnicodeCategory.ModifierLetter,
UnicodeCategory.OtherLetter,
UnicodeCategory.TitlecaseLetter,
UnicodeCategory.UppercaseLetter};
var RE_IDCHAR = RE.CharsOf(c => lettersCategories.Contains(Char.GetUnicodeCategory(c)));
var ID = global.DefineToken(RE_IDCHAR.Concat(
(RE_IDCHAR | RE.Range('0', '9')).Many()));
var NUM = global.DefineToken(RE.Range('0', '9').Many1());
var WHITESPACE = global.DefineToken(RE.Symbol(' ').Many());
var IF = keywords.DefineToken(RE.Literal("if"));
var ELSE = keywords.DefineToken(RE.Literal("else"));
var XMLNS = xml.DefineToken(RE.Literal("xmlns"));
var scannerInfo = lexicon.CreateScannerInfo();
scannerInfo.CurrentLexerIndex = xml.Index;
Scanner s = new Scanner(scannerInfo);
string source = "xmlns 你好吗1 123 蘏臦囧綗 AＢＣＤ if";
SourceReader sr = new SourceReader(new StringReader(source));
s.SetSource(sr);
s.SetTriviaTokens(WHITESPACE.Index);
var l1 = s.Read();
Assert.AreEqual(XMLNS.Index, l1.TokenIndex);
var l2 = s.Read();
Assert.AreEqual(ID.Index, l2.TokenIndex);
var l3 = s.Read();
Assert.AreEqual(NUM.Index, l3.TokenIndex);
var l4 = s.Read();
Assert.AreEqual(ID.Index, l4.TokenIndex);
var l5 = s.Read();
Assert.AreEqual(ID.Index, l5.TokenIndex);
var l6 = s.Read();
Assert.AreEqual(IF.Index, l6.TokenIndex);
}
public void SourceCodeTest()
{
string code = @"class ABCDEFG
{
public int c;
}";
StringReader sr = new StringReader(code);
SourceReader source = new SourceReader(sr);
Assert.AreEqual('c', (char)source.PeekChar());
Assert.AreEqual('c', (char)source.ReadChar());
Assert.AreEqual(0, source.Location.CharIndex);
//create a revert point
var rp1 = source.CreateRevertPoint();
Assert.AreEqual('l', (char)source.PeekChar());
Assert.AreEqual('l', (char)source.ReadChar());
Assert.AreEqual('a', (char)source.ReadChar());
Assert.AreEqual(2, source.Location.CharIndex);
//revert
source.Revert(rp1);
Assert.AreEqual('l', (char)source.PeekChar());
Assert.AreEqual('l', (char)source.ReadChar());
Assert.AreEqual('a', (char)source.ReadChar());
Assert.AreEqual(2, source.Location.CharIndex);
Assert.AreEqual('s', (char)source.ReadChar());
Assert.AreEqual(3, source.Location.CharIndex);
source.Revert(rp1);
source.RemoveRevertPoint(rp1);
Assert.AreEqual('l', (char)source.ReadChar());
Assert.AreEqual('a', (char)source.ReadChar());
Assert.AreEqual(2, source.Location.CharIndex);
Assert.AreEqual('s', (char)source.ReadChar());
Assert.AreEqual(3, source.Location.CharIndex);
Assert.AreEqual('s', (char)source.ReadChar());
Assert.Catch<ArgumentException>(() => source.Revert(rp1));
//peek and then revert
Assert.AreEqual(' ', (char)source.PeekChar());
var rp2 = source.CreateRevertPoint();
Assert.AreEqual(' ', (char)source.PeekChar());
Assert.AreEqual(' ', (char)source.ReadChar());
Assert.AreEqual('A', (char)source.ReadChar());
source.Revert(rp2);
Assert.AreEqual(' ', (char)source.PeekChar());
Assert.AreEqual(' ', (char)source.ReadChar());
Assert.AreEqual('A', (char)source.ReadChar());
//multiple revert point
var rp3 = source.CreateRevertPoint();
Assert.AreEqual('B', (char)source.ReadChar());
Assert.AreEqual('C', (char)source.ReadChar());
Assert.AreEqual('D', (char)source.ReadChar());
Assert.AreEqual('E', (char)source.PeekChar());
source.Revert(rp2);
Assert.AreEqual(' ', (char)source.PeekChar());
Assert.AreEqual(' ', (char)source.ReadChar());
Assert.AreEqual('A', (char)source.ReadChar());
source.Revert(rp3);
Assert.AreEqual('B', (char)source.ReadChar());
Assert.AreEqual('C', (char)source.ReadChar());
Assert.AreEqual('D', (char)source.ReadChar());
Assert.AreEqual('E', (char)source.PeekChar());
source.Revert(rp2);
Assert.AreEqual(' ', (char)source.PeekChar());
Assert.AreEqual(' ', (char)source.ReadChar());
Assert.AreEqual('A', (char)source.ReadChar());
source.RemoveRevertPoint(rp2);
source.RemoveRevertPoint(rp3);
Assert.AreEqual('B', (char)source.ReadChar());
Assert.AreEqual('C', (char)source.ReadChar());
Assert.AreEqual('D', (char)source.ReadChar());
Assert.AreEqual('E', (char)source.ReadChar());
Assert.AreEqual('F', (char)source.PeekChar());
}
public void Test(SourceReader sr)
{
Console.WriteLine("=============== CPS Parser Combinators ==================");
SetUpScanner();
var production = SetUpParser();
ForkableScannerBuilder fsb = new ForkableScannerBuilder(m_scannerInfo);
fsb.SetTriviaTokens(SPACE.Index);
var scanner = fsb.Create(sr);
var runner = new ParserRunner<int>(production);
try
{
var result = runner.Execute(scanner);
Console.WriteLine("Result: {0}", result);
}
catch (Exception ex)
{
Console.WriteLine("Parse Errors:");
Console.WriteLine(ex.Message);
}
Console.WriteLine();
Console.WriteLine();
}
public void Test(SourceReader sr)
{
Console.WriteLine("=============== Error Recovery Parser Combinators =======");
SetUpScanner();
var production = SetUpParser();
ForkableScannerBuilder fsb = new ForkableScannerBuilder(m_scannerInfo);
fsb.SetTriviaTokens(SPACE.Index);
var scanner = fsb.Create(sr);
var runner = new ParserRunner<int>(production);
List<SyntaxError> errors = new List<SyntaxError>();
var result = runner.Execute(scanner, errors);
if (errors.Count == 0)
{
Console.WriteLine("Result: {0}", result);
}
else
{
Console.WriteLine("Parse Errors:");
foreach (var err in errors)
{
Console.WriteLine("{0}:{1} {2}",
err.Code,
err.Description,
err.Location.StartLocation);
}
}
Console.WriteLine();
Console.WriteLine();
}
public void Test(SourceReader sr)
{
Console.WriteLine("=============== Simple Parser Combinators ===============");
SetUpScanner();
var parse = SetUpParser();
ForkableScannerBuilder fsb = new ForkableScannerBuilder(m_scannerInfo);
fsb.SetTriviaTokens(SPACE.Index);
var scanner = fsb.Create(sr);
try
{
var result = parse(scanner);
Console.WriteLine("Result: {0}", result.Value);
}
catch (Exception ex)
{
Console.WriteLine("Parse Errors:");
Console.WriteLine(ex.Message);
}
Console.WriteLine();
Console.WriteLine();
}
