public IList <IToken> GetTokens(TextFile sourceFile, out TimeSpan lexerTimeSpan)
{
SourceFile = sourceFile;
if (ErrorListener == null)
{
ErrorListener = new AntlrMemoryErrorListener();
ErrorListener.Logger = Logger;
ErrorListener.LineOffset = LineOffset;
}
ErrorListener.SourceFile = sourceFile;
var preprocessedText = PreprocessText(sourceFile);
AntlrInputStream inputStream;
if (Language.IsCaseInsensitive())
{
inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType);
}
else
{
inputStream = new AntlrInputStream(preprocessedText);
}
inputStream.name = sourceFile.RelativeName;
IList <IToken> tokens;
try
{
var stopwatch = Stopwatch.StartNew();
Lexer lexer = InitLexer(inputStream);
lexer.Interpreter = new LexerATNSimulator(lexer, GetOrCreateAtn(LexerSerializedATN));
lexer.RemoveErrorListeners();
lexer.AddErrorListener(ErrorListener);
tokens = lexer.GetAllTokens();
stopwatch.Stop();
lexerTimeSpan = stopwatch.Elapsed;
}
catch (Exception ex)
{
Logger.LogError(new LexingException(SourceFile, ex));
tokens = new List <IToken>();
}
finally
{
HandleMemoryConsumption();
}
return(tokens);
}
public ParseTree Parse(CodeFile sourceCodeFile)
{
AntlrParseTree result = null;
var filePath = sourceCodeFile.RelativeName;
if (sourceCodeFile.Code != null)
{
var errorListener = new AntlrMemoryErrorListener();
errorListener.CodeFile = sourceCodeFile;
errorListener.Logger = Logger;
errorListener.LineOffset = LineOffset;
try
{
var preprocessedText = PreprocessText(sourceCodeFile);
AntlrInputStream inputStream;
if (Language.IsCaseInsensitive)
{
inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType);
}
else
{
inputStream = new AntlrInputStream(preprocessedText);
}
inputStream.name = filePath;
Lexer lexer = InitLexer(inputStream);
lexer.Interpreter = new LexerATNSimulator(lexer, GetOrCreateAtn(true));
lexer.RemoveErrorListeners();
lexer.AddErrorListener(errorListener);
var commentTokens = new List <IToken>();
var stopwatch = Stopwatch.StartNew();
IList <IToken> tokens = lexer.GetAllTokens();
stopwatch.Stop();
long lexerTimeSpanTicks = stopwatch.ElapsedTicks;
foreach (IToken token in tokens)
{
if (token.Channel == CommentsChannel)
{
commentTokens.Add(token);
}
}
stopwatch.Restart();
var codeTokenSource = new ListTokenSource(tokens);
var codeTokenStream = new CommonTokenStream(codeTokenSource);
ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream);
stopwatch.Stop();
long parserTimeSpanTicks = stopwatch.ElapsedTicks;
result = Create(syntaxTree);
result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks);
result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks);
result.Tokens = tokens;
result.Comments = commentTokens;
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ParsingException(sourceCodeFile, ex));
if (result == null)
{
result = Create(null);
}
}
finally
{
Interlocked.Increment(ref processedFilesCount);
Interlocked.Add(ref processedBytesCount, sourceCodeFile.Code.Length);
}
}
else
{
result = Create(null);
}
result.SourceCodeFile = sourceCodeFile;
return(result);
}
protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile)
{
AntlrParseTree result = null;
var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name);
if (sourceCodeFile.Code != null)
{
var errorListener = new AntlrMemoryErrorListener();
errorListener.FileName = filePath;
errorListener.FileData = sourceCodeFile.Code;
errorListener.Logger = Logger;
errorListener.LineOffset = sourceCodeFile.LineOffset;
try
{
var preprocessedText = PreprocessText(sourceCodeFile);
AntlrInputStream inputStream;
if (Language.IsCaseInsensitive())
{
inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType);
}
else
{
inputStream = new AntlrInputStream(preprocessedText);
}
inputStream.name = filePath;
Lexer lexer = InitLexer(inputStream);
Lexer = lexer;
lexer.RemoveErrorListeners();
lexer.AddErrorListener(errorListener);
var commentTokens = new List <IToken>();
var stopwatch = Stopwatch.StartNew();
IList <IToken> tokens = GetAllTokens(lexer);
stopwatch.Stop();
long lexerTimeSpanTicks = stopwatch.ElapsedTicks;
#if DEBUG
var codeTokensStr = AntlrHelper.GetTokensString(tokens, Vocabulary, onlyDefaultChannel: false);
#endif
ClearLexerCacheIfRequired(lexer);
foreach (var token in tokens)
{
if (token.Channel == CommentsChannel)
{
commentTokens.Add(token);
}
}
stopwatch.Restart();
var codeTokenSource = new ListTokenSource(tokens);
var codeTokenStream = new CommonTokenStream(codeTokenSource);
ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream);
stopwatch.Stop();
long parserTimeSpanTicks = stopwatch.ElapsedTicks;
IncrementProcessedFilesCount();
result = Create(syntaxTree);
result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks);
result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks);
result.Tokens = tokens;
result.Comments = commentTokens;
}
catch (Exception ex)
{
Logger.LogError(new ParsingException(filePath, ex));
if (result == null)
{
result = Create(null);
}
}
}
else
{
result = Create(null);
}
result.FileName = filePath;
result.FileData = sourceCodeFile.Code;
return(result);
}