private void ProcessFile(string fileName, PatternMatcher patternMatcher, WorkflowResult workflowResult, CancellationToken cancellationToken = default(CancellationToken))
{
RootUst ust = null;
try
{
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingStarted, fileName));
ust = ReadParseAndConvert(fileName, workflowResult);
if (ust != null && Stage >= Stage.Match)
{
Stopwatch stopwatch = Stopwatch.StartNew();
IEnumerable <MatchResult> matchResults = patternMatcher.Match(ust);
stopwatch.Stop();
Logger.LogInfo($"File {ust.SourceCodeFile.Name} has been matched with patterns (Elapsed: {stopwatch.Elapsed}).");
workflowResult.AddMatchTime(stopwatch.ElapsedTicks);
workflowResult.AddResultEntity(matchResults);
cancellationToken.ThrowIfCancellationRequested();
RenderGraphs(workflowResult);
}
}
catch (OperationCanceledException)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogInfo($"{fileName} processing has been cancelled");
}
catch (ThreadAbortException)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogInfo(new OperationCanceledException($"Processing of {fileName} terimated due to depleted timeout ({FileTimeout})"));
Thread.ResetAbort();
}
catch (Exception ex)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogError(ex);
}
finally
{
AntlrParser.ClearCacheIfRequired();
workflowResult.AddProcessedFilesCount(1);
double progress = workflowResult.TotalFilesCount == 0
? workflowResult.TotalProcessedFilesCount
: (double)workflowResult.TotalProcessedFilesCount / workflowResult.TotalFilesCount;
Logger.LogInfo(new ProgressEventArgs(progress, fileName));
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingCompleted, fileName));
if (ust == null)
{
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingIgnored, fileName));
}
cancellationToken.ThrowIfCancellationRequested();
}
}
public override WorkflowResult Process(WorkflowResult workflowResult = null,
CancellationToken cancellationToken = default(CancellationToken))
{
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
BaseLanguages = GetBaseLanguages(AnalyzedLanguages);
var result = workflowResult ??
new WorkflowResult(AnalyzedLanguages.ToArray(), ThreadCount, Stage, IsIncludeIntermediateResult);
result.BaseLanguages = BaseLanguages.ToArray();
result.RenderStages = RenderStages;
result.IsSimplifyUst = IsSimplifyUst;
IEnumerable <string> fileNames = SourceCodeRepository.GetFileNames();
if (fileNames is IList <string> fileNamesList)
{
result.TotalFilesCount = fileNamesList.Count;
}
else
{
filesCountTask = Task.Factory.StartNew(() => result.TotalFilesCount = fileNames.Count());
}
try
{
var patternMatcher = new PatternMatcher
{
Logger = Logger,
Patterns = ConvertPatterns(result),
IsIgnoreFilenameWildcards = IsIgnoreFilenameWildcards
};
var parallelOptions = PrepareParallelOptions(cancellationToken);
Parallel.ForEach(
fileNames,
parallelOptions,
fileName =>
{
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
ProcessFileWithTimeout(fileName, patternMatcher, result, parallelOptions.CancellationToken);
});
}
catch (OperationCanceledException)
{
Logger.LogInfo("Scan has been cancelled");
}
if (result.TotalProcessedFilesCount > 1)
{
AntlrParser.ClearCacheIfRequired();
}
DumpJsonOutput(result);
result.ErrorCount = logger?.ErrorCount ?? 0;
return(result);
}
/// <summary>
/// Run the provided donatello program and return the result.
/// </summary>
/// <typeparam name="T">The expected return type</typeparam>
/// <param name="program">The donatello source code</param>
/// <returns>the return value of the program</returns>
public static T Run <T>(string program, string assemblyName = null, params string[] references)
{
const string namespaceName = "DonatelloRun";
const string className = "Runner";
const string methodName = "Run";
CompilationUnitSyntax syntaxTree = AntlrParser.ParseAsClass(program, namespaceName, className, methodName);
Stream result = Compiler.Compile(assemblyName ?? namespaceName, references, OutputType.DynamicallyLinkedLibrary, syntaxTree);
return(AssemblyRunner.Run <T>(result, namespaceName, className, methodName));
}
public override Root Run(string[] input)
{
var superComplexAst = AntlrParser.ParseAntlrAst(input);
var ast = AstSimplifier.Simplify(superComplexAst);
RunStep <ImportLinker>(ref ast);
RunStep <VariableAssigner>(ref ast);
RunStep <AstValidator>(ref ast);
return(ast);
}
public void Donatello_Transforms_ToCSharp(string test)
{
string file = Path.Combine("Unit", "Transforms", test);
string donatello = File.ReadAllText(file + ".dnl");
string expectedCSharp = File.ReadAllText(file + ".cs");
string actualCSharp = AntlrParser.ParseAsClass(donatello, "UnitTest", test + "Test")
.NormalizeWhitespace()
.ToFullString();
string normalizedExpectedCSharp = Whitespace.Replace(expectedCSharp, "");
string normalizedActualCSharp = Whitespace.Replace(actualCSharp, "");
Assert.Equal(normalizedExpectedCSharp, normalizedActualCSharp);
}
public Expression Parse(string bindingExpressionText, Type contextType, Type[] parentContexts, Type controlType)
{
try
{
var parser = new AntlrParser(bindingExpressionText);
parser.ExternalParameters = parser.ExternalParameters ?? new List<ParameterExpression>();
parser.ExternalParameters.AddRange(GetParameters(contextType, parentContexts, controlType));
parser.TypeRegistry = InitTypeRegistry(contextType, parentContexts);
var scope = Expression.Parameter(contextType, Constants.ThisSpecialBindingProperty);
return parser.Parse(scope);
}
catch (Exception exception)
{
throw new BidningParserExpception(contextType, bindingExpressionText, parentContexts, controlType, exception);
}
}
/// <summary>
/// Initial REPL setup and warmup
/// </summary>
private async static Task <ScriptState <object> > InitialEvaluationAsync()
{
// import required libraries
var usings = Compiler.DefaultImports.Keys
.Select(ns => $"using {ns};")
.StringJoin(" ");
var references = ScriptOptions.Default.WithReferences(Compiler.GetDefaultReferences());
var state = await CSharpScript.RunAsync(usings, references).ConfigureAwait(false);
// do a first run to "warm up" the repl so the user doesn't experience a delay for the first evaluation.
var warmup = AntlrParser.ParseAsRepl(@"(+ 30 12)")
.NormalizeWhitespace()
.ToFullString();
return(await state.ContinueWithAsync(warmup).ConfigureAwait(false));
}
public async Task RunAsync()
{
var initialState = Task.Run(InitialEvaluationAsync).ConfigureAwait(false);
DrawBanner();
ScriptState <object> state = null;
while (true)
{
//read!
Console.Write("> ");
string text = Console.ReadLine()?.Trim(); // null if ctrl-c
if (text == string.Empty)
{
continue;
}
if (text == "exit" || text == null)
{
break;
}
try
{
// eval!
// convert to roslyn tree
var program = AntlrParser.ParseAsRepl(text)
.NormalizeWhitespace()
.ToFullString();
// pass roslyn tree to scripting api
state = await(state ?? await initialState)
.ContinueWithAsync(program) // roslyn scripting api - continue program with existing instance state
.ConfigureAwait(false); // TPL api
// print!
ReplPrinter.Print(state.ReturnValue);
}
catch (Exception e)
{
Console.Error.WriteLine("Error: " + e.Message);
if (Debugger.IsAttached)
{
Console.Error.WriteLine(e.StackTrace);
}
}
} // loop!
}
public Program Parse()
{
var lexer = new AntlrLexer(new ANTLRStringStream(_code));
var tokens = new CommonTokenStream(lexer);
var parser = new AntlrParser(tokens);
parser.TreeAdaptor = new Adapter();
var result = parser.program();
lexer.Errors.ForEach(x => Errors.Add(x));
parser.Errors.ForEach(x => Errors.Add(x));
if (Errors.Any())
{
return(null);
}
var bridge = new BridgeVisitor(result.Tree);
return(bridge.CreateAst());
}
public void Initialize()
{
_parser = new AntlrParser();
}
