public void Start(CancellationToken token)
{
try
{
var code = RewriteAndPreprocess();
token.ThrowIfCancellationRequested();
var attributes = _attributeParser.Parse(_component);
token.ThrowIfCancellationRequested();
// temporal coupling... comments must be acquired before we walk the parse tree for declarations
// otherwise none of the annotations get associated to their respective Declaration
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _qualifiedName);
var stopwatch = Stopwatch.StartNew();
ITokenStream stream;
var tree = ParseInternal(code, new IParseTreeListener[]{ commentListener, annotationListener }, out stream);
stopwatch.Stop();
if (tree != null)
{
Debug.Print("IParseTree for component '{0}' acquired in {1}ms (thread {2})", _component.Name, stopwatch.ElapsedMilliseconds, Thread.CurrentThread.ManagedThreadId);
}
var comments = QualifyAndUnionComments(_qualifiedName, commentListener.Comments, commentListener.RemComments);
token.ThrowIfCancellationRequested();
ParseCompleted.Invoke(this, new ParseCompletionArgs
{
ParseTree = tree,
Tokens = stream,
Attributes = attributes,
Comments = comments,
Annotations = annotationListener.Annotations
});
}
catch (COMException exception)
{
Debug.WriteLine("Exception thrown in thread {0}:\n{1}", Thread.CurrentThread.ManagedThreadId, exception);
ParseFailure.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (SyntaxErrorException exception)
{
Debug.WriteLine("Exception thrown in thread {0}:\n{1}", Thread.CurrentThread.ManagedThreadId, exception);
ParseFailure.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (OperationCanceledException cancel)
{
Debug.WriteLine("Operation was Cancelled", cancel);
// no results to be used, so no results "returned"
//ParseCompleted.Invoke(this, new ParseCompletionArgs());
}
}
public void Start(CancellationToken token)
{
try
{
var code = RewriteAndPreprocess();
token.ThrowIfCancellationRequested();
var attributes = _attributeParser.Parse(_component);
token.ThrowIfCancellationRequested();
// temporal coupling... comments must be acquired before we walk the parse tree for declarations
// otherwise none of the annotations get associated to their respective Declaration
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _qualifiedName);
var stopwatch = Stopwatch.StartNew();
ITokenStream stream;
var tree = ParseInternal(_component.Name, code, new IParseTreeListener[] { commentListener, annotationListener }, out stream);
stopwatch.Stop();
if (tree != null)
{
_logger.Trace("IParseTree for component '{0}' acquired in {1}ms (thread {2})", _component.Name, stopwatch.ElapsedMilliseconds, Thread.CurrentThread.ManagedThreadId);
}
var comments = QualifyAndUnionComments(_qualifiedName, commentListener.Comments, commentListener.RemComments);
token.ThrowIfCancellationRequested();
ParseCompleted.Invoke(this, new ParseCompletionArgs
{
ParseTree = tree,
Tokens = stream,
Attributes = attributes,
Comments = comments,
Annotations = annotationListener.Annotations
});
}
catch (COMException exception)
{
_logger.Error(exception, "Exception thrown in thread {0}.", Thread.CurrentThread.ManagedThreadId);
ParseFailure.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (SyntaxErrorException exception)
{
_logger.Error(exception, "Exception thrown in thread {0}.", Thread.CurrentThread.ManagedThreadId);
ParseFailure.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (OperationCanceledException)
{
_logger.Debug("Component {0}: Operation was Cancelled", _component.Name);
// no results to be used, so no results "returned"
//ParseCompleted.Invoke(this, new ParseCompletionArgs());
}
}
private (IEnumerable <CommentNode> Comments, IEnumerable <IAnnotation> Annotations) CommentsAndAnnotations(QualifiedModuleName module, IParseTree tree)
{
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), module);
var combinedListener = new CombinedParseTreeListener(new IParseTreeListener[] { commentListener, annotationListener });
ParseTreeWalker.Default.Walk(combinedListener, tree);
var comments = QualifyAndUnionComments(module, commentListener.Comments, commentListener.RemComments);
var annotations = annotationListener.Annotations;
return(comments, annotations);
}
public static SourceFile BuildSourceFile(string name, string content, SourceMap.SourceMap?sourceMap,
Func <string, string, string> resolver)
{
AstToplevel toplevel;
AstSymbol? symbol;
if (PathUtils.GetExtension(name) == "json")
{
(toplevel, symbol) = Helpers.EmitVarDefineJson(content, name);
toplevel.FigureOutScope();
var exports = new StringTrie <AstNode>();
exports.Add(new ReadOnlySpan <string>(), symbol);
return(new SourceFile(name, toplevel)
{
Exports = exports
});
}
var commentListener = new CommentListener();
toplevel =
new Parser(new Options {
SourceFile = name, OnComment = commentListener.OnComment
}, content).Parse();
commentListener.Walk(toplevel);
sourceMap?.ResolveInAst(toplevel);
UnwrapIIFE(toplevel);
toplevel.FigureOutScope();
if (toplevel.Globals !.ContainsKey("module"))
{
(toplevel, symbol) = Helpers.IfPossibleEmitModuleExportsJsWrapper(toplevel);
if (symbol == null)
{
(toplevel, symbol) = Helpers.EmitCommonJsWrapper(toplevel);
}
toplevel.FigureOutScope();
var exports = new StringTrie <AstNode>();
exports.Add(new ReadOnlySpan <string>(), symbol);
var sourceFile = new SourceFile(name, toplevel)
{
Exports = exports,
OnlyWholeExport = true
};
sourceFile.Ast = (AstToplevel) new ImportExportTransformer(sourceFile, resolver).Transform(toplevel);
return(sourceFile);
}
public static SourceFile BuildSourceFile(string name, string content, SourceMap.SourceMap?sourceMap,
Func <string, string, string> resolver)
{
AstToplevel toplevel;
AstSymbol symbol;
if (PathUtils.GetExtension(name) == "json")
{
(toplevel, symbol) = Helpers.EmitVarDefineJson(content, name);
toplevel.FigureOutScope();
return(new SourceFile(name, toplevel)
{
WholeExport = symbol
});
}
var commentListener = new CommentListener();
toplevel =
new Parser(new Options {
SourceFile = name, OnComment = commentListener.OnComment
}, content).Parse();
commentListener.Walk(toplevel);
sourceMap?.ResolveInAst(toplevel);
toplevel.FigureOutScope();
if (toplevel.Globals !.ContainsKey("module"))
{
(toplevel, symbol) = Helpers.EmitCommonJsWrapper(toplevel);
toplevel.FigureOutScope();
return(new SourceFile(name, toplevel)
{
WholeExport = symbol
});
}
var sourceFile = new SourceFile(name, toplevel);
new ImportExportTransformer(sourceFile, resolver).Transform(toplevel);
return(sourceFile);
}
public void Start(CancellationToken cancellationToken)
{
try
{
Logger.Trace($"Starting ParseTaskID {_taskId} on thread {Thread.CurrentThread.ManagedThreadId}.");
var tokenStream = RewriteAndPreprocess(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
// temporal coupling... comments must be acquired before we walk the parse tree for declarations
// otherwise none of the annotations get associated to their respective Declaration
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _module);
var stopwatch = Stopwatch.StartNew();
var codePaneParseResults = ParseInternal(_module.ComponentName, tokenStream, new IParseTreeListener[] { commentListener, annotationListener });
stopwatch.Stop();
cancellationToken.ThrowIfCancellationRequested();
var comments = QualifyAndUnionComments(_module, commentListener.Comments, commentListener.RemComments);
cancellationToken.ThrowIfCancellationRequested();
var attributesPassParseResults = RunAttributesPass(cancellationToken);
var rewriter = new MemberAttributesRewriter(_exporter, _module.Component.CodeModule, new TokenStreamRewriter(attributesPassParseResults.tokenStream ?? tokenStream));
var completedHandler = ParseCompleted;
if (completedHandler != null && !cancellationToken.IsCancellationRequested)
{
completedHandler.Invoke(this, new ParseCompletionArgs
{
ParseTree = codePaneParseResults.tree,
AttributesTree = attributesPassParseResults.tree,
Tokens = codePaneParseResults.tokenStream,
AttributesRewriter = rewriter,
Attributes = attributesPassParseResults.attributes,
Comments = comments,
Annotations = annotationListener.Annotations
});
}
}
catch (COMException exception)
{
Logger.Error(exception, $"COM Exception thrown in thread {Thread.CurrentThread.ManagedThreadId} while parsing module {_module.ComponentName}, ParseTaskID {_taskId}.");
var failedHandler = ParseFailure;
failedHandler?.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (PreprocessorSyntaxErrorException syntaxErrorException)
{
var parsePassText = syntaxErrorException.ParsePass == ParsePass.CodePanePass
? "code pane"
: "exported";
Logger.Error($"Syntax error while preprocessing; offending token '{syntaxErrorException.OffendingSymbol.Text}' at line {syntaxErrorException.LineNumber}, column {syntaxErrorException.Position} in the {parsePassText} version of module {_module.ComponentName}.");
Logger.Debug(syntaxErrorException, $"SyntaxErrorException thrown in thread {Thread.CurrentThread.ManagedThreadId}, ParseTaskID {_taskId}.");
ReportException(syntaxErrorException);
}
catch (ParsePassSyntaxErrorException syntaxErrorException)
{
var parsePassText = syntaxErrorException.ParsePass == ParsePass.CodePanePass
? "code pane"
: "exported";
Logger.Error($"Syntax error; offending token '{syntaxErrorException.OffendingSymbol.Text}' at line {syntaxErrorException.LineNumber}, column {syntaxErrorException.Position} in the {parsePassText} version of module {_module.ComponentName}.");
Logger.Debug(syntaxErrorException, $"SyntaxErrorException thrown in thread {Thread.CurrentThread.ManagedThreadId}, ParseTaskID {_taskId}.");
ReportException(syntaxErrorException);
}
catch (SyntaxErrorException syntaxErrorException)
{
Logger.Error($"Syntax error; offending token '{syntaxErrorException.OffendingSymbol.Text}' at line {syntaxErrorException.LineNumber}, column {syntaxErrorException.Position} in module {_module.ComponentName}.");
Logger.Debug(syntaxErrorException, $"SyntaxErrorException thrown in thread {Thread.CurrentThread.ManagedThreadId}, ParseTaskID {_taskId}.");
ReportException(syntaxErrorException);
}
catch (OperationCanceledException exception)
{
//We report this, so that the calling code knows that the operation actually has been cancelled.
ReportException(exception);
}
catch (Exception exception)
{
Logger.Error(exception, $" Unexpected exception thrown in thread {Thread.CurrentThread.ManagedThreadId} while parsing module {_module.ComponentName}, ParseTaskID {_taskId}.");
ReportException(exception);
}
}
public void Start(CancellationToken token)
{
try
{
Logger.Trace("Starting ParseTaskID {0} on thread {1}.", _taskId, Thread.CurrentThread.ManagedThreadId);
var code = RewriteAndPreprocess(token);
token.ThrowIfCancellationRequested();
var attributes = _attributeParser.Parse(_component, token);
// temporal coupling... comments must be acquired before we walk the parse tree for declarations
// otherwise none of the annotations get associated to their respective Declaration
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _qualifiedName);
var stopwatch = Stopwatch.StartNew();
ITokenStream stream;
var tree = ParseInternal(_component.Name, code, new IParseTreeListener[] { commentListener, annotationListener }, out stream);
stopwatch.Stop();
token.ThrowIfCancellationRequested();
var comments = QualifyAndUnionComments(_qualifiedName, commentListener.Comments, commentListener.RemComments);
token.ThrowIfCancellationRequested();
var completedHandler = ParseCompleted;
if (completedHandler != null && !token.IsCancellationRequested)
{
completedHandler.Invoke(this, new ParseCompletionArgs
{
ParseTree = tree,
Tokens = stream,
Attributes = attributes,
Comments = comments,
Annotations = annotationListener.Annotations
});
}
}
catch (COMException exception)
{
Logger.Error(exception, "Exception thrown in thread {0}, ParseTaskID {1}.", Thread.CurrentThread.ManagedThreadId, _taskId);
var failedHandler = ParseFailure;
if (failedHandler != null)
{
failedHandler.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
}
catch (SyntaxErrorException exception)
{
Logger.Warn("Syntax error; offending token '{0}' at line {1}, column {2} in module {3}.", exception.OffendingSymbol.Text, exception.LineNumber, exception.Position, _qualifiedName);
Logger.Error(exception, "Exception thrown in thread {0}, ParseTaskID {1}.", Thread.CurrentThread.ManagedThreadId, _taskId);
var failedHandler = ParseFailure;
if (failedHandler != null)
{
failedHandler.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
}
catch (OperationCanceledException exception)
{
//We return this, so that the calling code knows that the operation actually has been cancelled.
var failedHandler = ParseFailure;
if (failedHandler != null)
{
failedHandler.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
}
catch (Exception exception)
{
Logger.Error(exception, "Exception thrown in thread {0}, ParseTaskID {1}.", Thread.CurrentThread.ManagedThreadId, _taskId);
var failedHandler = ParseFailure;
if (failedHandler != null)
{
failedHandler.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
}
}
public void Start(CancellationToken token)
{
try
{
Logger.Trace($"Starting ParseTaskID {_taskId} on thread {Thread.CurrentThread.ManagedThreadId}.");
var tokenStream = RewriteAndPreprocess(token);
token.ThrowIfCancellationRequested();
IParseTree attributesTree;
IDictionary <Tuple <string, DeclarationType>, Attributes> attributes;
var attributesTokenStream = RunAttributesPass(token, out attributesTree, out attributes);
var rewriter = new MemberAttributesRewriter(_exporter, _component.CodeModule, new TokenStreamRewriter(attributesTokenStream ?? tokenStream));
// temporal coupling... comments must be acquired before we walk the parse tree for declarations
// otherwise none of the annotations get associated to their respective Declaration
var commentListener = new CommentListener();
var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _qualifiedName);
var stopwatch = Stopwatch.StartNew();
ITokenStream stream;
var tree = ParseInternal(_component.Name, tokenStream, new IParseTreeListener[] { commentListener, annotationListener }, out stream);
stopwatch.Stop();
token.ThrowIfCancellationRequested();
var comments = QualifyAndUnionComments(_qualifiedName, commentListener.Comments, commentListener.RemComments);
token.ThrowIfCancellationRequested();
var completedHandler = ParseCompleted;
if (completedHandler != null && !token.IsCancellationRequested)
{
completedHandler.Invoke(this, new ParseCompletionArgs
{
ParseTree = tree,
AttributesTree = attributesTree,
Tokens = stream,
AttributesRewriter = rewriter,
Attributes = attributes,
Comments = comments,
Annotations = annotationListener.Annotations
});
}
}
catch (COMException exception)
{
Logger.Error(exception, $"Exception thrown in thread {Thread.CurrentThread.ManagedThreadId} while parsing module {_qualifiedName.Name}, ParseTaskID {_taskId}.");
var failedHandler = ParseFailure;
failedHandler?.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (SyntaxErrorException exception)
{
Logger.Warn($"Syntax error; offending token '{exception.OffendingSymbol.Text}' at line {exception.LineNumber}, column {exception.Position} in module {_qualifiedName}.");
Logger.Error(exception, $"Exception thrown in thread {Thread.CurrentThread.ManagedThreadId}, ParseTaskID {_taskId}.");
var failedHandler = ParseFailure;
failedHandler?.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (OperationCanceledException exception)
{
//We return this, so that the calling code knows that the operation actually has been cancelled.
var failedHandler = ParseFailure;
failedHandler?.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
catch (Exception exception)
{
Logger.Error(exception, $"Exception thrown in thread {Thread.CurrentThread.ManagedThreadId} while parsing module {_qualifiedName.Name}, ParseTaskID {_taskId}.");
var failedHandler = ParseFailure;
failedHandler?.Invoke(this, new ParseFailureArgs
{
Cause = exception
});
}
}
ParseTestCore(
ParserTestData testData)
{
string outAst;
var outMinJs = string.Empty;
var outMinJsMap = string.Empty;
var outNiceJs = string.Empty;
var outNiceJsMap = string.Empty;
try
{
var comments = new List <(bool block, string content, SourceLocation location)>();
var commentListener = new CommentListener();
var parser = new Parser(
new()
{
SourceFile = testData.SourceName, EcmaVersion = testData.EcmaScriptVersion, OnComment =
(block, content, location) =>
{
commentListener.OnComment(block, content, location);
comments.Add((block, content, location));
},
SourceType = testData.SourceName.StartsWith("module-") ? SourceType.Module : SourceType.Script
},
testData.Input);
var toplevel = parser.Parse();
commentListener.Walk(toplevel);
SourceMap?inputSourceMap = null;
if (testData.InputSourceMap != null)
{
inputSourceMap = SourceMap.Parse(testData.InputSourceMap, ".");
inputSourceMap.ResolveInAst(toplevel);
}
var strSink = new StringLineSink();
toplevel.FigureOutScope();
var dumper = new DumpAst(new AstDumpWriter(strSink));
dumper.Walk(toplevel);
foreach (var(block, content, location) in comments)
{
strSink.Print(
$"{(block ? "Block" : "Line")} Comment ({location.Start.ToShortString()}-{location.End.ToShortString()}): {content}");
}
outAst = strSink.ToString();
var outMinJsBuilder = new SourceMapBuilder();
var outputOptions = new OutputOptions
{
Shorthand = testData.EcmaScriptVersion >= 6
};
toplevel.PrintToBuilder(outMinJsBuilder, outputOptions);
outMinJsBuilder.AddText(
$"//# sourceMappingURL={PathUtils.ChangeExtension(testData.SourceName, "minjs.map")}");
if (inputSourceMap != null)
{
outMinJsBuilder.AttachSourcesContent(inputSourceMap);
}
outMinJs = outMinJsBuilder.Content();
outMinJsMap = outMinJsBuilder.Build(".", ".").ToString();
var outNiceJsBuilder = new SourceMapBuilder();
outputOptions = new()
{
Beautify = true,
Shorthand = testData.EcmaScriptVersion >= 6
};
toplevel.PrintToBuilder(outNiceJsBuilder, outputOptions);
outNiceJsBuilder.AddText(
$"//# sourceMappingURL={PathUtils.ChangeExtension(testData.SourceName, "nicejs.map")}");
if (inputSourceMap != null)
{
outNiceJsBuilder.AttachSourcesContent(inputSourceMap);
}
outNiceJs = outNiceJsBuilder.Content();
outNiceJsMap = outNiceJsBuilder.Build(".", ".").ToString();
strSink = new StringLineSink();
toplevel.FigureOutScope();
dumper = new DumpAst(new AstDumpWriter(strSink));
dumper.Walk(toplevel);
var beforeClone = strSink.ToString();
toplevel = toplevel.DeepClone();
strSink = new StringLineSink();
toplevel.FigureOutScope();
dumper = new DumpAst(new AstDumpWriter(strSink));
dumper.Walk(toplevel);
var afterClone = strSink.ToString();
if (beforeClone != afterClone)
{
throw new Exception("Dump of clone is not identical");
}
toplevel.Mangle();
}
catch (SyntaxError e)
{
outAst = e.Message;
}
return(outAst, outMinJs, outMinJsMap, outNiceJs, outNiceJsMap);
}
