/// <summary>
/// Main entry point to the CIL extractor.
/// Call this to extract a given assembly.
/// </summary>
/// <param name="layout">The trap layout.</param>
/// <param name="assemblyPath">The full path of the assembly to extract.</param>
/// <param name="logger">The logger.</param>
/// <param name="nocache">True to overwrite existing trap file.</param>
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
try
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new Extractor(false, assemblyPath, logger, pathTransformer);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (nocache || !System.IO.File.Exists(trapFile))
{
var cx = extractor.CreateContext(null, trapWriter, null, false);
ExtractCIL(cx, assemblyPath, extractPdbs);
extracted = true;
}
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
}
}
public void CanonicalPathUNCRoot()
{
CanonicalPathCache cache2 = CanonicalPathCache.Create(Logger, 1000, CanonicalPathCache.Symlinks.Preserve);
if (Win32.IsWindows())
{
var windows = cache.GetCanonicalPath(@"\WINDOWS").Replace(":", "$");
Assert.Equal($@"\\LOCALHOST\{windows}\bar", cache2.GetCanonicalPath($@"\\localhost\{windows}\bar"));
}
}
public CanonicalPathCacheTest()
{
File.Create("abc").Close();
cache = CanonicalPathCache.Create(Logger, 1000, CanonicalPathCache.Symlinks.Follow);
// Change directories to a directory that is in canonical form.
Directory.SetCurrentDirectory(cache.GetCanonicalPath(Path.GetTempPath()));
root = Win32.IsWindows() ? @"X:\" : "/";
}
public void CanonicalPathCacheSize()
{
cache = CanonicalPathCache.Create(Logger, 2, CanonicalPathCache.Symlinks.Preserve);
Assert.Equal(0, cache.CacheSize);
// The file "ABC" will fill the cache with parent directory info.
cache.GetCanonicalPath("ABC");
Assert.True(cache.CacheSize == 2);
string cp = cache.GetCanonicalPath("def");
Assert.Equal(2, cache.CacheSize);
Assert.Equal(Path.GetFullPath("def"), cp);
}
private static ExitCode AnalyseTracing(
TracingAnalyser analyser,
CSharpCommandLineArguments compilerArguments,
Options options,
CanonicalPathCache canonicalPathCache,
Stopwatch stopwatch)
{
return(Analyse(stopwatch, analyser, options,
references => ResolveReferences(compilerArguments, analyser, canonicalPathCache, references),
(analyser, syntaxTrees) =>
{
return ReadSyntaxTrees(
compilerArguments.SourceFiles.Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
},
(syntaxTrees, references) =>
{
// csc.exe (CSharpCompiler.cs) also provides CompilationOptions
// .WithMetadataReferenceResolver(),
// .WithXmlReferenceResolver() and
// .WithSourceReferenceResolver().
// These would be needed if we hadn't explicitly provided the source/references
// already.
return CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions
.WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default)
.WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
.WithMetadataImportOptions(MetadataImportOptions.All)
);
},
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
() => analyser.AnalyseCompilation(),
performance => analyser.LogPerformance(performance),
() => { }));
}
/// <summary>
/// Main entry point to the CIL extractor.
/// Call this to extract a given assembly.
/// </summary>
/// <param name="layout">The trap layout.</param>
/// <param name="assemblyPath">The full path of the assembly to extract.</param>
/// <param name="logger">The logger.</param>
/// <param name="nocache">True to overwrite existing trap file.</param>
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
try
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
using var trapWriter = transformedAssemblyPath.WithSuffix(".cil").CreateTrapWriter(logger, options.TrapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (!options.Cache || !System.IO.File.Exists(trapFile))
{
ExtractCIL(extractor, trapWriter, options.PDB);
extracted = true;
}
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
}
}
public static void ExtractStandalone(
IEnumerable <string> sources,
IEnumerable <string> referencePaths,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
try
{
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var commandLineArguments = Options.CreateWithEnvironment(args);
var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath());
var logger = commandLineArguments.Console
? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
using (var analyser = new Analyser(new LogProgressMonitor(logger), logger))
using (var references = new BlockingCollection <MetadataReference>())
{
try
{
var compilerVersion = new CompilerVersion(commandLineArguments);
bool preserveSymlinks = Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true";
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var cwd = Directory.GetCurrentDirectory();
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments == null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(
compilerArguments.SourceFiles.
Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = commandLineArguments.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
analyser.LogDiagnostics(compilerVersion.ArgsWithResponse);
return(ExitCode.Failed);
}
var compilation = CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions.
WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default).
WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
// csc.exe (CSharpCompiler.cs) also provides WithMetadataReferenceResolver,
// WithXmlReferenceResolver and
// WithSourceReferenceResolver.
// These would be needed if we hadn't explicitly provided the source/references
// already.
);
analyser.Initialize(compilerArguments, compilation, commandLineArguments, compilerVersion.ArgsWithResponse);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
sw.Restart();
analyser.PerformExtraction(commandLineArguments.Threads);
sw.Stop();
logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
}
/// <summary>
/// Construct tasks for resolving references (possibly in parallel).
///
/// The resolved references will be added (thread-safely) to the supplied
/// list <paramref name="ret"/>.
/// </summary>
static IEnumerable <Action> ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection <MetadataReference> ret)
{
var referencePaths = new Lazy <string[]>(() => FixedReferencePaths(args).ToArray());
return(args.MetadataReferences.Select <CommandLineReference, Action>(clref => () =>
{
if (Path.IsPathRooted(clref.Reference))
{
if (File.Exists(clref.Reference))
{
var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference));
ret.Add(reference);
}
else
{
lock (analyser)
{
analyser.Logger.Log(Severity.Error, " Reference '{0}' does not exist", clref.Reference);
++analyser.CompilationErrors;
}
}
}
else
{
bool referenceFound = false;
{
foreach (var composed in referencePaths.Value.
Select(path => Path.Combine(path, clref.Reference)).
Where(path => File.Exists(path)).
Select(path => canonicalPathCache.GetCanonicalPath(path)))
{
referenceFound = true;
var reference = MakeReference(clref, composed);
ret.Add(reference);
break;
}
if (!referenceFound)
{
lock (analyser)
{
analyser.Logger.Log(Severity.Error, " Unable to resolve reference '{0}'", clref.Reference);
++analyser.CompilationErrors;
}
}
}
}
}));
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var commandLineArguments = Options.CreateWithEnvironment(args);
var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath());
using var logger = commandLineArguments.Console
? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new Analyser(new LogProgressMonitor(logger), logger, commandLineArguments.AssemblySensitiveTrap, pathTransformer);
using var references = new BlockingCollection <MetadataReference>();
try
{
var compilerVersion = new CompilerVersion(commandLineArguments);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var cwd = Directory.GetCurrentDirectory();
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments == null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
{
logger.Log(Severity.Info, "Skipping extraction since files have already been extracted");
return(ExitCode.Ok);
}
var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(
compilerArguments.SourceFiles.
Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
var sw1 = new Stopwatch();
sw1.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = commandLineArguments.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
// csc.exe (CSharpCompiler.cs) also provides CompilationOptions
// .WithMetadataReferenceResolver(),
// .WithXmlReferenceResolver() and
// .WithSourceReferenceResolver().
// These would be needed if we hadn't explicitly provided the source/references
// already.
var compilation = CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions.
WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default).
WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
);
analyser.EndInitialize(compilerArguments, commandLineArguments, compilation);
analyser.AnalyseCompilation(cwd, args);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
var currentProcess = Process.GetCurrentProcess();
var cpuTime1 = currentProcess.TotalProcessorTime;
var userTime1 = currentProcess.UserProcessorTime;
sw1.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw1.Elapsed);
var sw2 = new Stopwatch();
sw2.Start();
analyser.PerformExtraction(commandLineArguments.Threads);
sw2.Stop();
var cpuTime2 = currentProcess.TotalProcessorTime;
var userTime2 = currentProcess.UserProcessorTime;
var performance = new Entities.PerformanceMetrics()
{
Frontend = new Entities.Timings()
{
Elapsed = sw1.Elapsed, Cpu = cpuTime1, User = userTime1
},
Extractor = new Entities.Timings()
{
Elapsed = sw2.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1
},
Total = new Entities.Timings()
{
Elapsed = stopwatch.Elapsed, Cpu = cpuTime2, User = userTime2
},
PeakWorkingSet = currentProcess.PeakWorkingSet64
};
analyser.LogPerformance(performance);
logger.Log(Severity.Info, " Extraction took {0}", sw2.Elapsed);
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
public static void ExtractStandalone(
IEnumerable <string> sources,
IEnumerable <string> referencePaths,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new Analyser(pm, logger, false, pathTransformer);
using var references = new BlockingCollection <MetadataReference>();
try
{
var referenceTasks = referencePaths.Select <string, Action>(path => () =>
{
var reference = MetadataReference.CreateFromFile(path);
references.Add(reference);
});
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = options.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
analyser.Logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
}
var compilation = CSharpCompilation.Create(
"csharp.dll",
syntaxTrees,
references
);
analyser.InitializeStandalone(compilation, options);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
sw.Restart();
analyser.PerformExtraction(options.Threads);
sw.Stop();
analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
foreach (var type in analyser.MissingNamespaces)
{
pm.MissingNamespace(type);
}
foreach (var type in analyser.MissingTypes)
{
pm.MissingType(type);
}
pm.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), args);
var options = Options.CreateWithEnvironment(Entities.Compilation.Settings.Args);
var fileLogger = new FileLogger(options.Verbosity, GetCSharpLogPath());
using var logger = options.Console
? new CombinedLogger(new ConsoleLogger(options.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !options.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, options.AssemblySensitiveTrap, pathTransformer);
try
{
if (options.ProjectsToLoad.Any())
{
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
}
var compilerVersion = new CompilerVersion(options);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
Entities.Compilation.Settings.Cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments is null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
{
logger.Log(Severity.Info, "Skipping extraction since files have already been extracted");
return(ExitCode.Ok);
}
return(AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch));
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
public void CanonicalPathPreserveLinksTests()
{
cache = CanonicalPathCache.Create(Logger, 1000, CanonicalPathCache.Symlinks.Preserve);
RunAllTests();
}
public void CanonicalPathFollowLinksTests()
{
cache = CanonicalPathCache.Create(Logger, 1000, CanonicalPathCache.Symlinks.Follow);
RunAllTests();
}
