public virtual void ParseAndRewrite()
{
ProcessArgs(args);
ICharStream input = null;
if (filename != null)
{
input = new ANTLRFileStream(filename);
}
else
{
input = new ANTLRReaderStream(Console.In, ANTLRReaderStream.InitialBufferSize, ANTLRReaderStream.ReadBufferSize);
}
// BUILD AST
ANTLRLexer lex = new ANTLRLexer(input);
tokens = new TokenRewriteStream(lex);
ANTLRParser g = new ANTLRParser(tokens);
Grammar grammar = new Grammar();
var r = g.grammar_(grammar);
CommonTree t = (CommonTree)r.Tree;
if (tree_option)
{
Console.Out.WriteLine(t.ToStringTree());
}
Rewrite(g.TreeAdaptor, t, g.TokenNames);
}
public virtual GrammarAST DupTree()
{
GrammarAST t = this;
ICharStream input = this.Token.InputStream;
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(input);
return((GrammarAST)adaptor.DupTree(t));
}
public virtual string ToTokenString()
{
ICharStream input = this.Token.InputStream;
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(input);
CommonTreeNodeStream nodes =
new CommonTreeNodeStream(adaptor, this);
StringBuilder buf = new StringBuilder();
GrammarAST o = (GrammarAST)nodes.LT(1);
int type = adaptor.GetType(o);
while (type != TokenTypes.EndOfFile)
{
buf.Append(" ");
buf.Append(o.Text);
nodes.Consume();
o = (GrammarAST)nodes.LT(1);
type = adaptor.GetType(o);
}
return(buf.ToString());
}
public virtual GrammarRootAST Parse(string fileName, ICharStream @in)
{
try
{
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(@in);
ToolANTLRLexer lexer = new ToolANTLRLexer(@in, this);
CommonTokenStream tokens = new CommonTokenStream(lexer);
lexer.tokens = tokens;
ToolANTLRParser p = new ToolANTLRParser(tokens, this);
p.TreeAdaptor = adaptor;
try
{
var r = p.grammarSpec();
GrammarAST root = (GrammarAST)r.Tree;
if (root is GrammarRootAST)
{
((GrammarRootAST)root).hasErrors = lexer.NumberOfSyntaxErrors > 0 || p.NumberOfSyntaxErrors > 0;
Debug.Assert(((GrammarRootAST)root).tokenStream == tokens);
if (grammarOptions != null)
{
((GrammarRootAST)root).cmdLineOptions = grammarOptions;
}
return((GrammarRootAST)root);
}
}
catch (v3TreeGrammarException e)
{
errMgr.GrammarError(ErrorType.V3_TREE_GRAMMAR, fileName, e.location);
}
return(null);
}
catch (RecognitionException)
{
// TODO: do we gen errors now?
ErrorManager.InternalError("can't generate this message at moment; antlr recovers");
}
return(null);
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
if (_failedTimeout)
{
return(classificationSpans);
}
bool spanExtended = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
ITextSnapshot snapshot = span.Snapshot;
ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState);
using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250)))
{
Span requestedSpan = span;
TState startState = AdjustParseSpan(classifierState, ref span);
ICharStream input = CreateInputStream(span);
ITokenSourceWithState <TState> lexer = CreateLexer(input, startState);
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
while (true)
{
IToken token = lexer.NextToken();
// The latter is true for EOF token with span.End at the end of the document
bool inBounds = token.StartIndex < span.End.Position ||
token.StopIndex < span.End.Position;
int startLineCurrent;
if (token.Type == CharStreamConstants.EndOfFile)
{
startLineCurrent = span.Snapshot.LineCount - 1;
}
else
{
startLineCurrent = token.Line - 1;
}
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF");
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
previousTokenEndsLine = endLinePrevious < startLineCurrent;
}
if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine))
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!classifierState._lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
for (int i = startLine; i < stopLine; i++)
{
if (!classifierState._lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (tokenEndsLine)
{
TState stateAtEndOfLine = lexer.GetCurrentState();
int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
lineStateChanged =
classifierState._lineStates[line].MultilineToken ||
!_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine));
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.Type == CharStreamConstants.EndOfFile)
{
break;
}
if (token.StartIndex >= span.End.Position)
{
break;
}
previousToken = token;
previousTokenEndsLine = tokenEndsLine;
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
// when considering the last line of a document, span and extendedSpan may end on the same line
ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine));
}
return(classificationSpans);
}
protected abstract ITokenSourceWithState <TState> CreateLexer(ICharStream input, TState startState);
public ToolANTLRLexer(ICharStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
public GrammarASTAdaptor(Antlr.Runtime.ICharStream input)
{
this.input = input;
}
public GrammarASTAdaptor(Antlr.Runtime.ICharStream input)
{
this.input = input;
}
public ToolANTLRLexer(ICharStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
public virtual GrammarRootAST Parse(string fileName, ICharStream @in)
{
try
{
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(@in);
ToolANTLRLexer lexer = new ToolANTLRLexer(@in, this);
CommonTokenStream tokens = new CommonTokenStream(lexer);
lexer.tokens = tokens;
ToolANTLRParser p = new ToolANTLRParser(tokens, this);
p.TreeAdaptor = adaptor;
try
{
var r = p.grammarSpec();
GrammarAST root = (GrammarAST)r.Tree;
if (root is GrammarRootAST)
{
((GrammarRootAST)root).hasErrors = lexer.NumberOfSyntaxErrors > 0 || p.NumberOfSyntaxErrors > 0;
Debug.Assert(((GrammarRootAST)root).tokenStream == tokens);
if (grammarOptions != null)
{
((GrammarRootAST)root).cmdLineOptions = grammarOptions;
}
return ((GrammarRootAST)root);
}
}
catch (v3TreeGrammarException e)
{
errMgr.GrammarError(ErrorType.V3_TREE_GRAMMAR, fileName, e.location);
}
return null;
}
catch (RecognitionException)
{
// TODO: do we gen errors now?
ErrorManager.InternalError("can't generate this message at moment; antlr recovers");
}
return null;
}
