public IListSource <LNode> Parse(ILexer input, IMessageSink msgs, Symbol inputType = null)
{
var preprocessed = new EcsPreprocessor(input);
var treeified = new TokensToTree(preprocessed, false);
return(Parse(treeified.Buffered(), input.SourceFile, msgs, inputType));
}
protected override void Stmt(string text, LNode expected, Action <EcsNodePrinter> configure = null, Mode mode = Mode.Both)
{
bool exprMode = (mode & Mode.Expression) != 0;
if ((mode & Mode.ParserTest) == 0)
{
return;
}
// This is the easy way:
//LNode result = EcsLanguageService.Value.ParseSingle(text, MessageSink.Console, exprMode ? ParsingService.Exprs : ParsingService.Stmts);
// But to make debugging easier, I'll do it the long way:
ILexer <Token> lexer = EcsLanguageService.Value.Tokenize(new UString(text), "", MessageSink.Console);
var preprocessed = new EcsPreprocessor(lexer);
var treeified = new TokensToTree(preprocessed, false);
var sink = (mode & Mode.ExpectAndDropParserError) != 0 ? new MessageHolder() : (IMessageSink)MessageSink.Console;
var parser = new EcsParser(treeified.Buffered(), lexer.SourceFile, sink);
LNode result = exprMode ? parser.ExprStart(false) : LNode.List(parser.ParseStmtsGreedy()).AsLNode(S.Splice);
AreEqual(TokenType.EOF, parser.LT0.Type(), string.Format("Parser stopped before EOF at [{0}] in {1}", parser.LT0.StartIndex, text));
AreEqual(expected, result);
if (sink is MessageHolder)
{
GreaterOrEqual(((MessageHolder)sink).List.Count, 1, "Expected an error but got none for " + text);
}
}
protected override void Stmt(string text, LNode expected, Action<EcsPrinterOptions> configure = null, Mode mode = Mode.Both)
{
bool exprMode = (mode & Mode.Expression) != 0;
if ((mode & Mode.ParserTest) == 0)
return;
var sink = (mode & Mode.ExpectAndDropParserError) != 0 ? new MessageHolder() : (IMessageSink)ConsoleMessageSink.Value;
// This is the easy way:
//LNode result = EcsLanguageService.Value.ParseSingle(text, sink, exprMode ? ParsingMode.Expressions : ParsingMode.Statements, preserveComments: true);
// But to make debugging easier, I'll do it the long way:
ILexer<Token> lexer = EcsLanguageService.Value.Tokenize(new UString(text), "", ConsoleMessageSink.Value);
var preprocessed = new EcsPreprocessor(lexer, true);
var treeified = new TokensToTree(preprocessed, false);
var parser = new EcsParser(treeified.Buffered(), lexer.SourceFile, sink);
VList<LNode> results = exprMode ? LNode.List(parser.ExprStart(false)) : LNode.List(parser.ParseStmtsGreedy());
//if (!preprocessed.TriviaList.IsEmpty)
{
// Inject comments
var injector = new EcsTriviaInjector(preprocessed.TriviaList, preprocessed.SourceFile, (int)TokenType.Newline, "/*", "*"+"/", "//");
results = LNode.List(injector.Run(results.GetEnumerator()).ToList());
}
LNode result = results.AsLNode(S.Splice);
AreEqual(TokenType.EOF, parser.LT0.Type(), string.Format("Parser stopped before EOF at [{0}] in {1}", parser.LT0.StartIndex, text));
if ((mode & Mode.IgnoreTrivia) != 0)
result = result.ReplaceRecursive(n => n.IsTrivia ? Maybe<LNode>.NoValue : n, LNode.ReplaceOpt.ProcessAttrs).Value;
AreEqual(expected, result);
if (sink is MessageHolder)
GreaterOrEqual(((MessageHolder)sink).List.Count, 1, "Expected an error but got none for "+text);
}
private IListSource <Token> ReadRestAsTokenTree()
{
ReadRest();
var restAsLexer = new TokenListAsLexer(_rest, _source.SourceFile);
var treeLexer = new TokensToTree(restAsLexer, false);
return(treeLexer.Buffered());
}
protected override void Stmt(string text, LNode expected, Action <EcsPrinterOptions> configure = null, Mode mode = Mode.Both)
{
bool exprMode = (mode & Mode.Expression) != 0;
if ((mode & (Mode.ParserTest | Mode.ExpectAndDropParserError)) == 0)
{
return;
}
var sink = (mode & Mode.ExpectAndDropParserError) != 0 ? new MessageHolder() : (IMessageSink)ConsoleMessageSink.Value;
using (Token.SetToStringStrategy(TokenExt.ToString)) // debugging aid
{
// This is the easy way:
// LNode result = EcsLanguageService.Value.ParseSingle(text, sink, exprMode ? ParsingMode.Expressions : ParsingMode.Statements, preserveComments: true);
// But to make debugging easier, I'll do it the long way:
ILexer <Token> lexer = EcsLanguageService.Value.Tokenize(new UString(text), "", sink);
var preprocessed = new EcsPreprocessor(lexer, true);
var treeified = new TokensToTree(preprocessed, false);
var parser = new EcsParser(treeified.Buffered(), lexer.SourceFile, sink, null);
LNodeList results = exprMode ? LNode.List(parser.ExprStart(false)) : LNode.List(parser.ParseStmtsGreedy());
// Inject comments
var injector = new EcsTriviaInjector(preprocessed.TriviaList, preprocessed.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", (mode & Mode.Expression) == 0);
results = LNode.List(injector.Run(results.GetEnumerator()).ToList());
LNode result = results.AsLNode(S.Splice);
AreEqual(TokenType.EOF, parser.LT0.Type(), string.Format("Parser stopped before EOF at [{0}] in {1}", parser.LT0.StartIndex, text));
if ((mode & Mode.IgnoreTrivia) != 0)
{
result = result.ReplaceRecursive(n => n.IsTrivia ? Maybe <LNode> .NoValue : n, LNode.ReplaceOpt.ProcessAttrs).Value;
}
if (sink is MessageHolder)
{
((MessageHolder)sink).WriteListTo(TraceMessageSink.Value);
GreaterOrEqual(((MessageHolder)sink).List.Count(m => m.Severity >= Severity.Error), 1,
"Expected an error but got none for " + text);
if (expected == null)
{
return;
}
}
if (!expected.Equals(result, LNode.CompareMode.TypeMarkers))
{
if ((mode & Mode.CompareAsLes) != 0)
{
using (LNode.SetPrinter(Syntax.Les.Les3LanguageService.Value))
AreEqual(expected.ToString(), result.ToString());
}
else
{
AreEqual(expected, result);
}
Fail("{0} has a different type marker than {1}", expected, result);
}
}
}
void TestStage1Core(string text, LNode expected)
{
var lexer = ParsingService.Default.Tokenize(text, ConsoleMessageSink.Value);
var treeified = new TokensToTree(lexer, true);
var tokens = treeified.Buffered();
var parser = new StageOneParser(tokens, lexer.SourceFile, ConsoleMessageSink.Value);
LNode result = parser.Parse();
AreEqual(expected, result);
}
public IListSource<LNode> Parse(ILexer<Token> input, IMessageSink msgs, ParsingMode inputType = null, bool preserveComments = true)
{
var preprocessed = new EcsPreprocessor(input, preserveComments);
var treeified = new TokensToTree(preprocessed, false);
var results = Parse(treeified.Buffered(), input.SourceFile, msgs, inputType);
if (preserveComments) {
var injector = new EcsTriviaInjector(preprocessed.TriviaList, input.SourceFile,
(int)TokenType.Newline, "/*", "*/", "//");
return injector.Run(results.GetEnumerator()).Buffered();
} else
return results;
}
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, ParsingMode inputType = null, bool preserveComments = true)
{
var preprocessed = new EcsPreprocessor(input, preserveComments);
var treeified = new TokensToTree(preprocessed, false);
var results = Parse(treeified.Buffered(), input.SourceFile, msgs, inputType);
if (preserveComments)
{
var injector = new EcsTriviaInjector(preprocessed.TriviaList, input.SourceFile,
(int)TokenType.Newline, "/*", "*/", "//");
return(injector.Run(results.GetEnumerator()).Buffered());
}
else
{
return(results);
}
}
public override IListSource <ITagSpan <ITag> > RunAnalysis(ITextSnapshot snapshot,
SparseAList <EditorToken> eTokens, CancellationToken cancellationToken)
{
var sourceFile = new TextSnapshotAsSourceFile(snapshot);
var tokens = ToNormalTokens(eTokens);
var tokensAsLexer = new TokenListAsLexer(tokens, sourceFile);
var tokensTree = new TokensToTree(new TokenListAsLexer(tokens, sourceFile), true)
{
ErrorSink = MessageSink.Trace
};
var results = new DList <ITagSpan <ClassificationTag> >();
var parser = new MyLesParser(tokensTree.Buffered(), sourceFile, MessageSink.Trace, results);
parser.ParseStmtsGreedy();
results.Sort((t1, t2) => t1.Span.Start.Position.CompareTo(t2.Span.Start.Position));
return(results);
}
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options)
{
var preprocessed = new EcsPreprocessor(input, options.PreserveComments);
var treeified = new TokensToTree(preprocessed, false);
var results = Parse(treeified.Buffered(), input.SourceFile, msgs, options);
if (options.PreserveComments)
{
var injector = new EcsTriviaInjector(preprocessed.TriviaList, input.SourceFile,
(int)TokenType.Newline, "/*", "*/", "//",
!options.Mode.IsOneOf <Symbol>(ParsingMode.Expressions, ParsingMode.FormalArguments));
return(injector.Run(results.GetEnumerator()).Buffered());
}
else
{
return(results);
}
}
protected override void Stmt(string text, LNode expected, Action<EcsNodePrinter> configure = null, Mode mode = Mode.Both)
{
bool exprMode = (mode & Mode.Expression) != 0;
if ((mode & Mode.ParserTest) == 0)
return;
// This is the easy way:
//LNode result = EcsLanguageService.Value.ParseSingle(text, MessageSink.Console, exprMode ? ParsingService.Exprs : ParsingService.Stmts);
// But to make debugging easier, I'll do it the long way:
ILexer<Token> lexer = EcsLanguageService.Value.Tokenize(new UString(text), "", MessageSink.Console);
var preprocessed = new EcsPreprocessor(lexer);
var treeified = new TokensToTree(preprocessed, false);
var sink = (mode & Mode.ExpectAndDropParserError) != 0 ? new MessageHolder() : (IMessageSink)MessageSink.Console;
var parser = new EcsParser(treeified.Buffered(), lexer.SourceFile, sink);
LNode result = exprMode ? parser.ExprStart(false) : parser.Stmt();
AreEqual(TokenType.EOF, parser.LT0.Type());
AreEqual(expected, result);
if (sink is MessageHolder)
GreaterOrEqual(((MessageHolder)sink).List.Count, 1);
}
public IListSource <LNode> Parse(ILexer input, IMessageSink msgs, Symbol inputType = null)
{
var treeified = new TokensToTree(input, true);
return(Parse(treeified.Buffered(), input.SourceFile, msgs, inputType));
}
public IListSource<LNode> Parse(ILexer<Token> input, IMessageSink msgs, Symbol inputType = null)
{
var preprocessed = new EcsPreprocessor(input);
var treeified = new TokensToTree(preprocessed, false);
return Parse(treeified.Buffered(), input.SourceFile, msgs, inputType);
}
protected override void Stmt(string text, LNode expected, Action<EcsNodePrinter> configure = null, bool exprMode = false, Mode mode = Mode.Both)
{
if (mode == Mode.PrintOnly)
return;
// This is the easy way:
//LNode result = EcsLanguageService.Value.ParseSingle(text, MessageSink.Console, exprMode ? ParsingService.Exprs : ParsingService.Stmts);
// But to make debugging easier, I'll do it the long way:
ILexer<Token> lexer = EcsLanguageService.Value.Tokenize(new UString(text), "", MessageSink.Console);
var preprocessed = new EcsPreprocessor(lexer);
var treeified = new TokensToTree(preprocessed, false);
var parser = new EcsParser(treeified.Buffered(), lexer.SourceFile, MessageSink.Console);
LNode result = exprMode ? parser.ExprStart(false) : parser.Stmt();
AreEqual(TokenType.EOF, parser.LT0.Type());
AreEqual(expected, result);
}
void TestStage1Core(string text, LNode expected)
{
var lexer = ParsingService.Default.Tokenize(text, ConsoleMessageSink.Value);
var treeified = new TokensToTree(lexer, true);
var tokens = treeified.Buffered();
var parser = new StageOneParser(tokens, lexer.SourceFile, ConsoleMessageSink.Value);
LNode result = parser.Parse();
AreEqual(expected, result);
}
public IListSource<LNode> Parse(ILexer<Token> input, IMessageSink msgs, Symbol inputType = null)
{
var treeified = new TokensToTree(input, true);
return Parse(treeified.Buffered(), input.SourceFile, msgs, inputType);
}