/// <summary>
/// Construct tasks for reading source code files (possibly in parallel).
///
/// The constructed syntax trees will be added (thread-safely) to the supplied
/// list <paramref name="ret"/>.
/// </summary>
static IEnumerable <Action> ReadSyntaxTrees(IEnumerable <string> sources, Analyser analyser, CSharpParseOptions parseOptions, Encoding encoding, IList <SyntaxTree> ret)
{
return(sources.Select <string, Action>(path => () =>
{
try
{
using (var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var st = CSharpSyntaxTree.ParseText(SourceText.From(file, encoding), parseOptions, path);
lock (ret)
ret.Add(st);
}
}
catch (IOException ex)
{
lock (analyser)
{
analyser.Logger.Log(Severity.Error, " Unable to open source file {0}: {1}", path, ex.Message);
++analyser.CompilationErrors;
}
}
}));
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var commandLineArguments = Options.CreateWithEnvironment(args);
var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath());
var logger = commandLineArguments.Console
? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
using (var analyser = new Analyser(new LogProgressMonitor(logger), logger))
using (var references = new BlockingCollection <MetadataReference>())
{
try
{
var compilerVersion = new CompilerVersion(commandLineArguments);
bool preserveSymlinks = Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true";
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var cwd = Directory.GetCurrentDirectory();
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments == null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(
compilerArguments.SourceFiles.
Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = commandLineArguments.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
analyser.LogDiagnostics(compilerVersion.ArgsWithResponse);
return(ExitCode.Failed);
}
var compilation = CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions.
WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default).
WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
// csc.exe (CSharpCompiler.cs) also provides WithMetadataReferenceResolver,
// WithXmlReferenceResolver and
// WithSourceReferenceResolver.
// These would be needed if we hadn't explicitly provided the source/references
// already.
);
analyser.Initialize(compilerArguments, compilation, commandLineArguments, compilerVersion.ArgsWithResponse);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
sw.Restart();
analyser.PerformExtraction(commandLineArguments.Threads);
sw.Stop();
logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
}
public static void ExtractStandalone(
IEnumerable <string> sources,
IEnumerable <string> referencePaths,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
using (var analyser = new Analyser(pm, logger))
using (var references = new BlockingCollection <MetadataReference>())
{
try
{
var referenceTasks = referencePaths.Select <string, Action>(path => () =>
{
var reference = MetadataReference.CreateFromFile(path);
references.Add(reference);
});
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = options.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
analyser.Logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
}
var compilation = CSharpCompilation.Create(
"csharp.dll",
syntaxTrees,
references
);
analyser.InitializeStandalone(compilation, options);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
sw.Restart();
analyser.PerformExtraction(options.Threads);
sw.Stop();
analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
foreach (var type in analyser.MissingNamespaces)
{
pm.MissingNamespace(type);
}
foreach (var type in analyser.MissingTypes)
{
pm.MissingType(type);
}
pm.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
}
/// <summary>
/// Construct tasks for resolving references (possibly in parallel).
///
/// The resolved references will be added (thread-safely) to the supplied
/// list <paramref name="ret"/>.
/// </summary>
static IEnumerable <Action> ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection <MetadataReference> ret)
{
var referencePaths = new Lazy <string[]>(() => FixedReferencePaths(args).ToArray());
return(args.MetadataReferences.Select <CommandLineReference, Action>(clref => () =>
{
if (Path.IsPathRooted(clref.Reference))
{
if (File.Exists(clref.Reference))
{
var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference));
ret.Add(reference);
}
else
{
lock (analyser)
{
analyser.Logger.Log(Severity.Error, " Reference '{0}' does not exist", clref.Reference);
++analyser.CompilationErrors;
}
}
}
else
{
bool referenceFound = false;
{
foreach (var composed in referencePaths.Value.
Select(path => Path.Combine(path, clref.Reference)).
Where(path => File.Exists(path)).
Select(path => canonicalPathCache.GetCanonicalPath(path)))
{
referenceFound = true;
var reference = MakeReference(clref, composed);
ret.Add(reference);
break;
}
if (!referenceFound)
{
lock (analyser)
{
analyser.Logger.Log(Severity.Error, " Unable to resolve reference '{0}'", clref.Reference);
++analyser.CompilationErrors;
}
}
}
}
}));
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var commandLineArguments = Options.CreateWithEnvironment(args);
var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath());
using var logger = commandLineArguments.Console
? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new Analyser(new LogProgressMonitor(logger), logger, commandLineArguments.AssemblySensitiveTrap, pathTransformer);
using var references = new BlockingCollection <MetadataReference>();
try
{
var compilerVersion = new CompilerVersion(commandLineArguments);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var cwd = Directory.GetCurrentDirectory();
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments == null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
{
logger.Log(Severity.Info, "Skipping extraction since files have already been extracted");
return(ExitCode.Ok);
}
var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(
compilerArguments.SourceFiles.
Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
var sw1 = new Stopwatch();
sw1.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = commandLineArguments.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
// csc.exe (CSharpCompiler.cs) also provides CompilationOptions
// .WithMetadataReferenceResolver(),
// .WithXmlReferenceResolver() and
// .WithSourceReferenceResolver().
// These would be needed if we hadn't explicitly provided the source/references
// already.
var compilation = CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions.
WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default).
WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
);
analyser.EndInitialize(compilerArguments, commandLineArguments, compilation);
analyser.AnalyseCompilation(cwd, args);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
var currentProcess = Process.GetCurrentProcess();
var cpuTime1 = currentProcess.TotalProcessorTime;
var userTime1 = currentProcess.UserProcessorTime;
sw1.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw1.Elapsed);
var sw2 = new Stopwatch();
sw2.Start();
analyser.PerformExtraction(commandLineArguments.Threads);
sw2.Stop();
var cpuTime2 = currentProcess.TotalProcessorTime;
var userTime2 = currentProcess.UserProcessorTime;
var performance = new Entities.PerformanceMetrics()
{
Frontend = new Entities.Timings()
{
Elapsed = sw1.Elapsed, Cpu = cpuTime1, User = userTime1
},
Extractor = new Entities.Timings()
{
Elapsed = sw2.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1
},
Total = new Entities.Timings()
{
Elapsed = stopwatch.Elapsed, Cpu = cpuTime2, User = userTime2
},
PeakWorkingSet = currentProcess.PeakWorkingSet64
};
analyser.LogPerformance(performance);
logger.Log(Severity.Info, " Extraction took {0}", sw2.Elapsed);
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
private static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOptions options,
Func <BlockingCollection <MetadataReference>, IEnumerable <Action> > getResolvedReferenceTasks,
Func <Analyser, List <SyntaxTree>, IEnumerable <Action> > getSyntaxTreeTasks,
Func <IEnumerable <SyntaxTree>, IEnumerable <MetadataReference>, CSharpCompilation> getCompilation,
Action <CSharpCompilation, CommonOptions> initializeAnalyser,
Action analyseCompilation,
Action <Entities.PerformanceMetrics> logPerformance,
Action postProcess)
{
using var references = new BlockingCollection <MetadataReference>();
var referenceTasks = getResolvedReferenceTasks(references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = getSyntaxTreeTasks(analyser, syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = options.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
analyser.Logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
if (analyser is TracingAnalyser)
{
return(ExitCode.Failed);
}
}
var compilation = getCompilation(syntaxTrees, references);
initializeAnalyser(compilation, options);
analyseCompilation();
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
var elapsed = sw.Elapsed;
var currentProcess = Process.GetCurrentProcess();
var cpuTime1 = currentProcess.TotalProcessorTime;
var userTime1 = currentProcess.UserProcessorTime;
sw.Restart();
analyser.PerformExtraction(options.Threads);
sw.Stop();
var cpuTime2 = currentProcess.TotalProcessorTime;
var userTime2 = currentProcess.UserProcessorTime;
var performance = new Entities.PerformanceMetrics()
{
Frontend = new Entities.Timings()
{
Elapsed = elapsed, Cpu = cpuTime1, User = userTime1
},
Extractor = new Entities.Timings()
{
Elapsed = sw.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1
},
Total = new Entities.Timings()
{
Elapsed = stopwatch.Elapsed, Cpu = cpuTime2, User = userTime2
},
PeakWorkingSet = currentProcess.PeakWorkingSet64
};
logPerformance(performance);
analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
postProcess();
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
