/// <summary>
/// Main entry point to the CIL extractor.
/// Call this to extract a given assembly.
/// </summary>
/// <param name="layout">The trap layout.</param>
/// <param name="assemblyPath">The full path of the assembly to extract.</param>
/// <param name="logger">The logger.</param>
/// <param name="nocache">True to overwrite existing trap file.</param>
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
try
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new Extractor(false, assemblyPath, logger, pathTransformer);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (nocache || !System.IO.File.Exists(trapFile))
{
var cx = extractor.CreateContext(null, trapWriter, null, false);
ExtractCIL(cx, assemblyPath, extractPdbs);
extracted = true;
}
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
}
}
public void TestTransformerFile()
{
var spec = new string[]
{
@"#D:\src",
@"C:\agent*\src//",
@"-C:\agent*\src\external",
@"",
@"#empty",
@"",
@"#src2",
@"/agent*//src",
@"",
@"#optsrc",
@"opt/src//"
};
var pathTransformer = new PathTransformer(new PathCacheStub(), spec);
// Windows-style matching
Assert.Equal(@"C:/bar.cs", pathTransformer.Transform(@"C:\bar.cs").Value);
Assert.Equal("D:/src/file.cs", pathTransformer.Transform(@"C:\agent42\src\file.cs").Value);
Assert.Equal("D:/src/file.cs", pathTransformer.Transform(@"C:\agent43\src\file.cs").Value);
Assert.Equal(@"C:/agent43/src/external/file.cs", pathTransformer.Transform(@"C:\agent43\src\external\file.cs").Value);
// Linux-style matching
Assert.Equal(@"src2/src/file.cs", pathTransformer.Transform(@"/agent/src/file.cs").Value);
Assert.Equal(@"src2/src/file.cs", pathTransformer.Transform(@"/agent42/src/file.cs").Value);
Assert.Equal(@"optsrc/file.cs", pathTransformer.Transform(@"/opt/src/file.cs").Value);
}
public void LeftSidePath_FromRightAbsolutPath()
{
PathTransformer pathTransformer = new PathTransformer("http://somedomain/", "http://egoratest/somedomain/");
string newPath = pathTransformer.AdjustPath("/page1.html");
Assert.IsNotNull(newPath);
Assert.AreEqual("http://egoratest/somedomain/page1.html", newPath);
}
public void LocationHeaderTest()
{
string location =
"http://localhost:9090/fkb2local/views/client.api.DatenImportierenLandesIT.jsf?FT=X";
PathTransformer transformer = new PathTransformer("http://localhost:9090/fkb2local/", "/bmi.gv.at/fk2web-t/");
Assert.AreEqual("/bmi.gv.at/fk2web-t/views/client.api.DatenImportierenLandesIT.jsf?FT=X", transformer.AdjustPath(location));
}
/// <summary>
/// Extract an assembly to a new trap file.
/// If the trap file exists, skip extraction to avoid duplicating
/// extraction within the snapshot.
/// </summary>
/// <param name="r">The assembly to extract.</param>
private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
{
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var assemblyPath = r.FilePath;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: true);
var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
if (!skipExtraction)
{
/* Note on parallel builds:
*
* The trap writer and source archiver both perform atomic moves
* of the file to the final destination.
*
* If the same source file or trap file are generated concurrently
* (by different parallel invocations of the extractor), then
* last one wins.
*
* Specifically, if two assemblies are analysed concurrently in a build,
* then there is a small amount of duplicated work but the output should
* still be correct.
*/
// compilation.Clone() reduces memory footprint by allowing the symbols
// in c to be garbage collected.
Compilation c = compilation.Clone();
if (c.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
{
var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath), AddAssemblyTrapPrefix);
foreach (var module in assembly.Modules)
{
AnalyseNamespace(cx, module.GlobalNamespace);
}
Entities.Attribute.ExtractAttributes(cx, assembly, Extraction.Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
cx.PopulateAll();
}
}
ReportProgress(assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", r.FilePath, ex);
}
}
public void LeftSidePath_FromRightAbsolutPathAndNoRootProxyPath()
{
PathTransformer pathTransformer = new PathTransformer("https://www.fundamt.gv.at/gondor/",
"http://egoratest/stammportal/fundamt/");
string newPath = pathTransformer.AdjustPath("/gondor/WebGov/UI/Security/FormsLogin.aspx");
Assert.IsNotNull(newPath);
Assert.AreEqual("http://egoratest/stammportal/fundamt/WebGov/UI/Security/FormsLogin.aspx", newPath);
}
public Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
{
Logger = logger;
AddAssemblyTrapPrefix = addAssemblyTrapPrefix;
Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
stopWatch.Start();
progressMonitor = pm;
PathTransformer = pathTransformer;
}
public void LocationHeaderTest2()
{
string location =
"http://localhost:9090/fkb2local/views/client.api.DatenImportierenLandesIT.jsf?FT=X";
string expectedLaoction = location;
PathTransformer transformer = new PathTransformer("https://pvawp.bmi.gv.at/bmi.gv.at/fk2web-t/", "/bmi.gv.at/fk2web-t/blabla.jsp");
Assert.AreEqual(expectedLaoction, transformer.AdjustPath(location));
}
public void LeftSidePath_FromRightAbsolutPathAndApplicationAsSubdir()
{
PathTransformer pathTransformer =
new PathTransformer("https://awp.statistik.at/statistik.at/vis.test.extern/",
"/statistik.at/vis.test.extern/");
string newPath = pathTransformer.AdjustPath("https://awp.statistik.at/statistik.at/vis.test.extern/start.do");
Assert.IsNotNull(newPath);
Assert.AreEqual("/statistik.at/vis.test.extern/start.do", newPath);
}
public MockRedisContentLocationStoreFactory(
ITestRedisDatabase redisDatabase,
ITestRedisDatabase machineLocationRedisDatabase,
AbsolutePath localCacheRoot,
IClock mockClock = null,
RedisContentLocationStoreConfiguration configuration = null)
{
LocalMachineData = PathTransformer.GetLocalMachineLocation(localCacheRoot);
_mockClock = mockClock ?? SystemClock.Instance;
RedisDatabase = redisDatabase;
MachineLocationRedisDatabase = machineLocationRedisDatabase;
_configuration = configuration ?? RedisContentLocationStoreConfiguration.Default;
}
public string TransformsToExpectedResult(string pathToTransform)
{
var originalPathsFromManifest = new string[]
{
"Spartan.exe",
"art\\Art.bar",
"sound\\SoundSets\\Amb.xml"
};
var builder = new PathTransformerBuilder();
var transformer = new PathTransformer(builder.Build(originalPathsFromManifest));
return(transformer.TransformPath(pathToTransform));
}
private void DoExtractTree(SyntaxTree tree)
{
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var sourcePath = tree.FilePath;
var transformedSourcePath = PathTransformer.Transform(sourcePath);
var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
var excluded = projectLayout == null;
var trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
var upToDate = false;
if (!excluded)
{
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false);
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
if (!upToDate)
{
var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), AddAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Extraction.Entities.File.Create(cx, root.SyntaxTree.FilePath);
var csNode = (CSharpSyntaxNode)root;
csNode.Accept(new CompilationUnitVisitor(cx));
csNode.Accept(new DirectiveVisitor(cx));
cx.PopulateAll();
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
cx.PopulateAll();
}
}
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded
? AnalysisAction.Excluded
: upToDate
? AnalysisAction.UpToDate
: AnalysisAction.Extracted);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
}
}
private void DoAnalyseCompilation()
{
try
{
var assemblyPath = extractor.OutputPath;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var assembly = compilation.Assembly;
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), AddAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
}
}
private void DoExtractTree(SyntaxTree tree)
{
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var sourcePath = tree.FilePath;
var transformedSourcePath = PathTransformer.Transform(sourcePath);
var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
var excluded = projectLayout == null;
var trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
var upToDate = false;
if (!excluded)
{
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false);
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
if (!upToDate)
{
var cx = extractor.CreateContext(compilation.Clone(), trapWriter, new SourceScope(tree), AddAssemblyTrapPrefix);
Populators.CompilationUnit.Extract(cx, tree.GetRoot());
cx.PopulateAll();
cx.ExtractComments(cx.CommentGenerator);
cx.PopulateAll();
}
}
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded
? AnalysisAction.Excluded
: upToDate
? AnalysisAction.UpToDate
: AnalysisAction.Extracted);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
extractor.Message(new Message("Unhandled exception processing syntax tree", tree.FilePath, null, ex.StackTrace));
}
}
/// <summary>
/// Main entry point to the CIL extractor.
/// Call this to extract a given assembly.
/// </summary>
/// <param name="layout">The trap layout.</param>
/// <param name="assemblyPath">The full path of the assembly to extract.</param>
/// <param name="logger">The logger.</param>
/// <param name="nocache">True to overwrite existing trap file.</param>
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
try
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
using var trapWriter = transformedAssemblyPath.WithSuffix(".cil").CreateTrapWriter(logger, options.TrapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (!options.Cache || !System.IO.File.Exists(trapFile))
{
ExtractCIL(extractor, trapWriter, options.PDB);
extracted = true;
}
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
}
}
public static void ExtractStandalone(
IEnumerable <string> sources,
IEnumerable <string> referencePaths,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
try
{
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), args);
var options = Options.CreateWithEnvironment(Entities.Compilation.Settings.Args);
var fileLogger = new FileLogger(options.Verbosity, GetCSharpLogPath());
using var logger = options.Console
? new CombinedLogger(new ConsoleLogger(options.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !options.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, options.AssemblySensitiveTrap, pathTransformer);
try
{
if (options.ProjectsToLoad.Any())
{
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
}
var compilerVersion = new CompilerVersion(options);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
Entities.Compilation.Settings.Cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments is null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
{
logger.Log(Severity.Info, "Skipping extraction since files have already been extracted");
return(ExitCode.Ok);
}
return(AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch));
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
public static void ExtractStandalone(
IEnumerable <string> sources,
IEnumerable <string> referencePaths,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new Analyser(pm, logger, false, pathTransformer);
using var references = new BlockingCollection <MetadataReference>();
try
{
var referenceTasks = referencePaths.Select <string, Action>(path => () =>
{
var reference = MetadataReference.CreateFromFile(path);
references.Add(reference);
});
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees);
var sw = new Stopwatch();
sw.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = options.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
analyser.Logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
}
var compilation = CSharpCompilation.Create(
"csharp.dll",
syntaxTrees,
references
);
analyser.InitializeStandalone(compilation, options);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
sw.Stop();
analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
sw.Restart();
analyser.PerformExtraction(options.Threads);
sw.Stop();
analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
foreach (var type in analyser.MissingNamespaces)
{
pm.MissingNamespace(type);
}
foreach (var type in analyser.MissingTypes)
{
pm.MissingType(type);
}
pm.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
/// <summary>
/// Command-line driver for the extractor.
/// </summary>
///
/// <remarks>
/// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a
/// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully
/// drive Roslyn in the way that csc.exe would.
/// </remarks>
///
/// <param name="args">Command line arguments as passed to csc.exe</param>
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var commandLineArguments = Options.CreateWithEnvironment(args);
var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath());
using var logger = commandLineArguments.Console
? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger)
: (ILogger)fileLogger;
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer)
{
logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer");
return(ExitCode.Ok);
}
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new Analyser(new LogProgressMonitor(logger), logger, commandLineArguments.AssemblySensitiveTrap, pathTransformer);
using var references = new BlockingCollection <MetadataReference>();
try
{
var compilerVersion = new CompilerVersion(commandLineArguments);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return(ExitCode.Ok);
}
var cwd = Directory.GetCurrentDirectory();
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
cwd,
compilerVersion.FrameworkPath,
compilerVersion.AdditionalReferenceDirectories
);
if (compilerArguments == null)
{
var sb = new StringBuilder();
sb.Append(" Failed to parse command line: ").AppendList(" ", args);
logger.Log(Severity.Error, sb.ToString());
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
{
logger.Log(Severity.Info, "Skipping extraction since files have already been extracted");
return(ExitCode.Ok);
}
var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references);
var syntaxTrees = new List <SyntaxTree>();
var syntaxTreeTasks = ReadSyntaxTrees(
compilerArguments.SourceFiles.
Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,
syntaxTrees);
var sw1 = new Stopwatch();
sw1.Start();
Parallel.Invoke(
new ParallelOptions {
MaxDegreeOfParallelism = commandLineArguments.Threads
},
referenceTasks.Interleave(syntaxTreeTasks).ToArray());
if (syntaxTrees.Count == 0)
{
logger.Log(Severity.Error, " No source files");
++analyser.CompilationErrors;
return(ExitCode.Failed);
}
// csc.exe (CSharpCompiler.cs) also provides CompilationOptions
// .WithMetadataReferenceResolver(),
// .WithXmlReferenceResolver() and
// .WithSourceReferenceResolver().
// These would be needed if we hadn't explicitly provided the source/references
// already.
var compilation = CSharpCompilation.Create(
compilerArguments.CompilationName,
syntaxTrees,
references,
compilerArguments.CompilationOptions.
WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default).
WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths))
);
analyser.EndInitialize(compilerArguments, commandLineArguments, compilation);
analyser.AnalyseCompilation(cwd, args);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
{
analyser.AnalyseTree(tree);
}
var currentProcess = Process.GetCurrentProcess();
var cpuTime1 = currentProcess.TotalProcessorTime;
var userTime1 = currentProcess.UserProcessorTime;
sw1.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw1.Elapsed);
var sw2 = new Stopwatch();
sw2.Start();
analyser.PerformExtraction(commandLineArguments.Threads);
sw2.Stop();
var cpuTime2 = currentProcess.TotalProcessorTime;
var userTime2 = currentProcess.UserProcessorTime;
var performance = new Entities.PerformanceMetrics()
{
Frontend = new Entities.Timings()
{
Elapsed = sw1.Elapsed, Cpu = cpuTime1, User = userTime1
},
Extractor = new Entities.Timings()
{
Elapsed = sw2.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1
},
Total = new Entities.Timings()
{
Elapsed = stopwatch.Elapsed, Cpu = cpuTime2, User = userTime2
},
PeakWorkingSet = currentProcess.PeakWorkingSet64
};
analyser.LogPerformance(performance);
logger.Log(Severity.Info, " Extraction took {0}", sw2.Elapsed);
return(analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
return(ExitCode.Errors);
}
}
public static async Task <bool> ScanAndRepairFile(GameFileInfo fileInfo, string gameFilePath,
PathTransformer pathTransformer, IProgress <ScanSubProgress> progress = null,
int concurrentDownload = 0, CancellationToken ct = default)
{
var filePath = Path.Combine(gameFilePath, pathTransformer.TransformPath(fileInfo.FileName));
//#1 Check File
ct.ThrowIfCancellationRequested();
progress?.Report(new ScanSubProgress(ScanSubProgressStep.Check, 0));
Progress <double> subProgressCheck = null;
if (progress != null)
{
subProgressCheck = new Progress <double>();
subProgressCheck.ProgressChanged += (o, d) =>
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.Check, d));
};
}
if (await RunFileCheck(filePath, fileInfo.Size, fileInfo.Crc32, ct, subProgressCheck))
{
progress?.Report(new ScanSubProgress(ScanSubProgressStep.End, 100));
return(true);
}
//#2 Download File
ct.ThrowIfCancellationRequested();
progress?.Report(new ScanSubProgress(ScanSubProgressStep.Download, 0));
var tempFileName = Path.Combine(GameScannerTempPath, $"{fileInfo.FileName.GetHashCode():X4}.tmp");
if (File.Exists(tempFileName))
{
File.Delete(tempFileName);
}
var fileDownloader = concurrentDownload == 1
? (IFileDownloader) new SimpleFileDownloader(fileInfo.HttpLink, tempFileName)
: new ChunkFileDownloader(fileInfo.HttpLink, tempFileName, GameScannerTempPath,
concurrentDownload);
if (progress != null)
{
fileDownloader.ProgressChanged += (sender, eventArg) =>
{
switch (fileDownloader.State)
{
case FileDownloaderState.Invalid:
case FileDownloaderState.Download:
progress.Report(new ScanSubProgress(
ScanSubProgressStep.Download, fileDownloader.DownloadProgress * 0.99,
new ScanDownloadProgress(fileDownloader.DownloadSize, fileDownloader.BytesDownloaded,
fileDownloader.DownloadSpeed)));
break;
case FileDownloaderState.Finalize:
progress.Report(new ScanSubProgress(
ScanSubProgressStep.Download, 99,
new ScanDownloadProgress(fileDownloader.DownloadSize, fileDownloader.BytesDownloaded,
0)));
break;
case FileDownloaderState.Complete:
progress.Report(new ScanSubProgress(
ScanSubProgressStep.Download, 100,
new ScanDownloadProgress(fileDownloader.DownloadSize, fileDownloader.BytesDownloaded,
0)));
break;
case FileDownloaderState.Error:
case FileDownloaderState.Abort:
break;
default:
throw new ArgumentOutOfRangeException(nameof(fileDownloader.State),
fileDownloader.State, null);
}
}
}
;
await fileDownloader.DownloadAsync(ct);
//#3 Check Downloaded File
ct.ThrowIfCancellationRequested();
Progress <double> subProgressCheckDown = null;
if (progress != null)
{
progress.Report(new ScanSubProgress(ScanSubProgressStep.CheckDownload, 0));
subProgressCheckDown = new Progress <double>();
subProgressCheckDown.ProgressChanged += (o, d) =>
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.CheckDownload, d));
};
}
try
{
await EnsureValidGameFile(tempFileName, fileInfo.BinSize, fileInfo.BinCrc32, ct, subProgressCheckDown);
}
catch
{
if (File.Exists(tempFileName))
{
File.Delete(tempFileName);
}
throw;
}
//#4 Extract downloaded file
ct.ThrowIfCancellationRequested();
if (L33TZipUtils.IsL33TZipFile(tempFileName))
{
var tempFileName2 = $"{tempFileName.Replace(".tmp", string.Empty)}.ext.tmp";
//
Progress <double> extractProgress = null;
if (progress != null)
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.ExtractDownload, 0));
extractProgress = new Progress <double>();
extractProgress.ProgressChanged += (o, d) =>
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.ExtractDownload, d));
};
}
await L33TZipUtils.ExtractL33TZipFileAsync(tempFileName, tempFileName2, ct, extractProgress);
//#4.1 Check Extracted File
ct.ThrowIfCancellationRequested();
Progress <double> subProgressCheckExt = null;
if (progress != null)
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.CheckExtractDownload, 0));
subProgressCheckExt = new Progress <double>();
subProgressCheckExt.ProgressChanged += (o, d) =>
{
progress.Report(new ScanSubProgress(
ScanSubProgressStep.CheckExtractDownload, d));
};
}
await EnsureValidGameFile(tempFileName2, fileInfo.Size, fileInfo.Crc32, ct, subProgressCheckExt);
File.Delete(tempFileName);
tempFileName = tempFileName2;
}
//#5 Move new file to game folder
ct.ThrowIfCancellationRequested();
progress?.Report(new ScanSubProgress(
ScanSubProgressStep.Finalize, 0));
if (File.Exists(filePath))
{
File.Delete(filePath);
}
else
{
var pathName = Path.GetDirectoryName(filePath);
if (!string.IsNullOrEmpty(pathName) && !Directory.Exists(pathName))
{
Directory.CreateDirectory(pathName);
}
}
File.Move(tempFileName, filePath);
//#6 End
progress?.Report(new ScanSubProgress(
ScanSubProgressStep.End, 100));
return(true);
}
public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
: base(pm, logger, addAssemblyTrapPrefix, pathTransformer)
{
}
