public void ParserFuncTest()
{
Lexicon test = new Lexicon();
var ID = test.DefaultLexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.DefaultLexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.DefaultLexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.DefaultLexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER.AsParser(l => l.PrefixTrivia.Count == 0)
from n in NUM
select "A";
var p2 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "B";
var parser1 = p1 | p2;
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetTriviaTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
ParserRunner <string> runner = new ParserRunner <string>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("A", result1);
Assert.AreEqual(0, errorManager.Errors.Count);
string source2 = "abc > > 123";
var sr2 = new SourceReader(new StringReader(source2));
ForkableScanner scanner2 = builder.Create(sr2);
var result2 = runner.Run(scanner2);
Assert.AreEqual("B", result2);
Assert.AreEqual(0, errorManager.Errors.Count);
}
public void ForkableScannerTest()
{
Lexicon lexicon = new Lexicon();
var A = lexicon.DefaultLexer.DefineToken(RE.Range('a', 'z'));
ScannerInfo si = lexicon.CreateScannerInfo();
string source = "abcdefghijklmnopqrstuvwxyz";
ForkableScannerBuilder fsBuilder = new ForkableScannerBuilder(si);
ForkableScanner fscanner = fsBuilder.Create(new SourceReader(new StringReader(source)));
var l1 = fscanner.Read();
Assert.AreEqual("a", l1.Value);
var l2 = fscanner.Read();
Assert.AreEqual("b", l2.Value);
//fork
ForkableScanner fscanner2 = fscanner.Fork();
for (int i = 2; i <= 4; i++)
{
var l = fscanner.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
for (int i = 2; i <= 5; i++)
{
var l = fscanner2.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
ForkableScanner fscanner3 = fscanner.Fork();
var l5a = fscanner.Read();
var l5b = fscanner3.Read();
Assert.AreEqual(source[5].ToString(), l5a.Value);
Assert.AreEqual(source[5].ToString(), l5b.Value);
var l6b = fscanner2.Read();
var l6a = fscanner3.Read();
Assert.AreEqual(source[6].ToString(), l6a.Value);
Assert.AreEqual(source[6].ToString(), l6b.Value);
var l7a = fscanner2.Read();
for (int i = 7; i < 9; i++)
{
var l = fscanner3.Read();
Assert.AreEqual(source[i].ToString(), l.Value);
}
}
public T Parse(SourceReader source)
{
CodeContract.RequiresArgumentNotNull(source, "source");
if (!m_isInitialized)
{
OnInitialize();
}
ForkableScanner scanner = m_scannerBuilder.Create(source);
return(m_parserRunner.Run(scanner));
}
public static IResult <Lexeme> Scan(this Token token, ForkableScanner scanner)
{
var lexeme = scanner.Read();
if (lexeme.TokenIndex == token.Index)
{
return(new Result <Lexeme>(lexeme));
}
else
{
return(null);
}
}
public ForkableScanner Create(SourceReader source)
{
CodeContract.RequiresArgumentNotNull(source, "source");
Scanner masterScanner = new Scanner(ScannerInfo);
masterScanner.SetSource(source);
masterScanner.SetTriviaTokens(m_triviaTokens);
masterScanner.ErrorList = ErrorList;
masterScanner.RecoverErrors = RecoverErrors;
masterScanner.LexicalErrorId = LexicalErrorId;
return(ForkableScanner.Create(masterScanner));
}
public T Parse(SourceReader source, CompilationErrorList errorList)
{
CodeContract.RequiresArgumentNotNull(source, "source");
if (!m_isInitialized)
{
OnInitialize();
}
m_scannerBuilder.ErrorList = errorList;
m_context.ErrorList = errorList;
ForkableScanner scanner = m_scannerBuilder.Create(source);
return(m_parserRunner.Run(scanner));
}
public void ParserCastTest()
{
Lexicon test = new Lexicon();
var ID = test.Lexer.DefineToken(RE.Range('a', 'z').Concat(
(RE.Range('a', 'z') | RE.Range('0', '9')).Many()));
var NUM = test.Lexer.DefineToken(RE.Range('0', '9').Many1());
var GREATER = test.Lexer.DefineToken(RE.Symbol('>'));
var WHITESPACE = test.Lexer.DefineToken(RE.Symbol(' ').Union(RE.Symbol('\t')));
var p1 = from i in ID
from g in GREATER
from g2 in GREATER
from n in NUM
select "hello";
var parser1 = p1.TryCast <object>();
var info = test.CreateScannerInfo();
ForkableScannerBuilder builder = new ForkableScannerBuilder(info);
builder.SetTriviaTokens(WHITESPACE.Index);
var errorManager = new CompilationErrorManager();
var context = new ParserContext(errorManager, 1, 2);
context.DefineDefaultCompilationErrorInfo(0);
var el = errorManager.CreateErrorList();
context.ErrorList = el;
ParserRunner <object> runner = new ParserRunner <object>(parser1, context);
string source1 = "abc >> 123";
var sr1 = new SourceReader(new StringReader(source1));
ForkableScanner scanner1 = builder.Create(sr1);
var result1 = runner.Run(scanner1);
Assert.AreEqual("hello", result1);
Assert.AreEqual(0, el.Count);
}
public override Parse <TFuture> BuildParse <TFuture>(Future <string, TFuture> future)
{
Parse <TFuture> parse = null;
parse = scanner =>
{
// 保存读取当前单词之前的位置
ForkableScanner prevScanner = scanner.Fork();
Lexeme l = scanner.Read();
if (l.TokenIndex == m_token.Index)
{
return(new StepResult <TFuture>(0, () => future(l.Value.Content)(scanner)));
}
Lexeme recovery = l.GetErrorCorrectionLexeme(m_token.Index, m_token.Description);
SyntaxError insertionErr = Grammar.RecoverByInsertion(recovery);
if (l.IsEndOfStream)
{
// 已经到了输入的末尾
// 插入预期的Token进行恢复
return(new StepResult <TFuture>(1,
() => future(recovery.Value.Content)(prevScanner), insertionErr));
}
else
{
// 同时尝试插入预期的Token，以及删除当前字符
// 在未来的解析中选取更好的路线
return(Grammar.Best(
new StepResult <TFuture>(1, () => future(recovery.Value.Content)(prevScanner), insertionErr),
new StepResult <TFuture>(1, () => parse(scanner), Grammar.RecoverByDeletion(l))));
}
};
return(parse);
}
public T Run(ForkableScanner scanner)
{
var result = m_runner(scanner, Context);
return(result.GetResult(Context));
}
public Result(T value, ForkableScanner rest)
{
Value = value;
Rest = rest;
}
public T Execute(ForkableScanner scanner, IList <SyntaxError> errors)
{
var result = m_parseFunc(scanner);
return(result.GetResult(errors));
}
public T Execute(ForkableScanner scanner)
{
var result = m_parseFunc(scanner);
return(result.GetResult());
}
