private int Run()
{
InitializeDefaultOptions();
ProcessCommandLine();
if (_commandLineOptions.OutputFilePath == null)
{
throw new CommandLineException("Output filename must be specified (/out <file>)");
}
//
// Set target Architecture and OS
//
if (_commandLineOptions.TargetArch != null)
{
if (_commandLineOptions.TargetArch.Equals("x86", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X86;
}
else if (_commandLineOptions.TargetArch.Equals("x64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X64;
}
else if (_commandLineOptions.TargetArch.Equals("arm", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM;
}
else if (_commandLineOptions.TargetArch.Equals("armel", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM;
}
else if (_commandLineOptions.TargetArch.Equals("arm64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM64;
}
else
{
throw new CommandLineException("Target architecture is not supported");
}
}
if (_commandLineOptions.TargetOS != null)
{
if (_commandLineOptions.TargetOS.Equals("windows", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Windows;
}
else if (_commandLineOptions.TargetOS.Equals("linux", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Linux;
}
else if (_commandLineOptions.TargetOS.Equals("osx", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.OSX;
}
else
{
throw new CommandLineException("Target OS is not supported");
}
}
using (PerfEventSource.StartStopEvents.CompilationEvents())
{
ICompilation compilation;
using (PerfEventSource.StartStopEvents.LoadingEvents())
{
//
// Initialize type system context
//
SharedGenericsMode genericsMode = SharedGenericsMode.CanonicalReferenceTypes;
var targetDetails = new TargetDetails(_targetArchitecture, _targetOS, TargetAbi.CoreRT, SimdVectorLength.None);
CompilerTypeSystemContext typeSystemContext = new ReadyToRunCompilerContext(targetDetails, genericsMode);
//
// TODO: To support our pre-compiled test tree, allow input files that aren't managed assemblies since
// some tests contain a mixture of both managed and native binaries.
//
// See: https://github.com/dotnet/corert/issues/2785
//
// When we undo this this hack, replace this foreach with
// typeSystemContext.InputFilePaths = _inputFilePaths;
//
Dictionary <string, string> inputFilePaths = new Dictionary <string, string>();
foreach (var inputFile in _inputFilePaths)
{
try
{
var module = typeSystemContext.GetModuleFromPath(inputFile.Value);
inputFilePaths.Add(inputFile.Key, inputFile.Value);
}
catch (TypeSystemException.BadImageFormatException)
{
// Keep calm and carry on.
}
}
typeSystemContext.InputFilePaths = inputFilePaths;
typeSystemContext.ReferenceFilePaths = _referenceFilePaths;
string systemModuleName = _commandLineOptions.SystemModule ?? DefaultSystemModule;
typeSystemContext.SetSystemModule(typeSystemContext.GetModuleForSimpleName(systemModuleName));
if (typeSystemContext.InputFilePaths.Count == 0)
{
throw new CommandLineException("No input files specified");
}
//
// Initialize compilation group and compilation roots
//
// Single method mode?
MethodDesc singleMethod = CheckAndParseSingleMethodModeArguments(typeSystemContext);
var logger = new Logger(Console.Out, _commandLineOptions.Verbose);
List <ModuleDesc> referenceableModules = new List <ModuleDesc>();
foreach (var inputFile in inputFilePaths)
{
try
{
referenceableModules.Add(typeSystemContext.GetModuleFromPath(inputFile.Value));
}
catch { } // Ignore non-managed pe files
}
foreach (var referenceFile in _referenceFilePaths.Values)
{
try
{
referenceableModules.Add(typeSystemContext.GetModuleFromPath(referenceFile));
}
catch { } // Ignore non-managed pe files
}
ProfileDataManager profileDataManager = new ProfileDataManager(logger, referenceableModules);
CompilationModuleGroup compilationGroup;
List <ICompilationRootProvider> compilationRoots = new List <ICompilationRootProvider>();
if (singleMethod != null)
{
// Compiling just a single method
compilationGroup = new SingleMethodCompilationModuleGroup(singleMethod);
compilationRoots.Add(new SingleMethodRootProvider(singleMethod));
}
else
{
// Either single file, or multifile library, or multifile consumption.
EcmaModule entrypointModule = null;
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
if (module.PEReader.PEHeaders.IsExe)
{
if (entrypointModule != null)
{
throw new Exception("Multiple EXE modules");
}
entrypointModule = module;
}
}
List <EcmaModule> inputModules = new List <EcmaModule>();
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
compilationRoots.Add(new ReadyToRunRootProvider(module, profileDataManager));
inputModules.Add(module);
if (!_commandLineOptions.InputBubble)
{
break;
}
}
List <ModuleDesc> versionBubbleModules = new List <ModuleDesc>();
if (_commandLineOptions.InputBubble)
{
// In large version bubble mode add reference paths to the compilation group
foreach (string referenceFile in _referenceFilePaths.Values)
{
try
{
// Currently SimpleTest.targets has no easy way to filter out non-managed assemblies
// from the reference list.
EcmaModule module = typeSystemContext.GetModuleFromPath(referenceFile);
versionBubbleModules.Add(module);
}
catch (TypeSystemException.BadImageFormatException ex)
{
Console.WriteLine("Warning: cannot open reference assembly '{0}': {1}", referenceFile, ex.Message);
}
}
}
compilationGroup = new ReadyToRunSingleAssemblyCompilationModuleGroup(
typeSystemContext, inputModules, versionBubbleModules, _commandLineOptions.CompileBubbleGenerics,
_commandLineOptions.Partial ? profileDataManager : null);
}
//
// Compile
//
string inputFilePath = "";
foreach (var input in typeSystemContext.InputFilePaths)
{
inputFilePath = input.Value;
break;
}
ReadyToRunCodegenCompilationBuilder builder = new ReadyToRunCodegenCompilationBuilder(typeSystemContext, compilationGroup, inputFilePath);
string compilationUnitPrefix = "";
builder.UseCompilationUnitPrefix(compilationUnitPrefix);
ILProvider ilProvider = new ReadyToRunILProvider();
DependencyTrackingLevel trackingLevel = _commandLineOptions.DgmlLogFileName == null ?
DependencyTrackingLevel.None : (_commandLineOptions.GenerateFullDgmlLog ? DependencyTrackingLevel.All : DependencyTrackingLevel.First);
builder
.UseIbcTuning(_commandLineOptions.Tuning)
.UseResilience(_commandLineOptions.Resilient)
.UseMapFile(_commandLineOptions.Map)
.UseParallelism(_commandLineOptions.Parallelism)
.UseILProvider(ilProvider)
.UseJitPath(_commandLineOptions.JitPath)
.UseBackendOptions(_commandLineOptions.CodegenOptions)
.UseLogger(logger)
.UseDependencyTracking(trackingLevel)
.UseCompilationRoots(compilationRoots)
.UseOptimizationMode(_optimizationMode);
compilation = builder.ToCompilation();
}
compilation.Compile(_commandLineOptions.OutputFilePath.FullName);
if (_commandLineOptions.DgmlLogFileName != null)
{
compilation.WriteDependencyLog(_commandLineOptions.DgmlLogFileName.FullName);
}
}
return(0);
}
public CopiedDebugDirectoryEntryNode(EcmaModule sourceModule, int debugEntryIndex)
: base(sourceModule)
{
Debug.Assert(debugEntryIndex >= 0);
_debugEntryIndex = debugEntryIndex;
}
public NativeDebugDirectoryEntryNode(EcmaModule sourceModule)
: base(sourceModule)
{
}
public void AddInputModule(EcmaModule module)
{
_inputModules.Add(module);
}
public TypesTableNode(TargetDetails target, EcmaModule module)
: base(target)
{
_module = module;
}
public IBCModule(EcmaModule ecmaModule, Dictionary <IBCBlobKey, BlobEntry> blobs)
{
EcmaModule = ecmaModule;
Blobs = blobs;
}
private MethodDesc GetSigMethodInstantiationFromIBCMethodSpec(IBCModule ibcModule, BlobReader sig)
{
TypeDesc methodType = GetSigTypeFromIBCZapSig(ibcModule, ibcModule.EcmaModule, sig);
SkipTypeInIBCZapSig(ref sig);
uint flags = (uint)sig.ReadCompressedInteger();
if (Macros.IsSlotUsedInsteadOfToken(flags))
{
int slot = sig.ReadCompressedInteger();
_logger.Writer.WriteLine($"Warning: IBC Data for `{methodType}` with slot '{slot}' was ignored");
return(null); // Unsupported case thought to be used only for array methods, which don't really matter for R2R codegen
}
else
{
// Decode method token
uint methodRid = (uint)sig.ReadCompressedInteger();
uint methodToken;
MetadataType methodMetadataType = (MetadataType)methodType;
if (ibcModule.EcmaModule == methodMetadataType.Module)
{
methodToken = Cor.Macros.TokenFromRid(methodRid, CorTokenType.mdtMethodDef);
}
else
{
uint ibcToken = Cor.Macros.TokenFromRid(methodRid, CorTokenType.ibcExternalMethod);
methodToken = LookupIbcMethodToken(methodMetadataType, ibcToken, ibcModule.Blobs);
if (Cor.Macros.RidFromToken(methodToken) == 0)
{
_logger.Writer.WriteLine($"Warning: External Method Token {ibcToken:x} on '{methodMetadataType}' could not be found.");
return(null);
}
}
EcmaModule ecmaModuleOfMethod = ((EcmaType)methodMetadataType.GetTypeDefinition()).EcmaModule;
MethodDesc ecmaMethod = ecmaModuleOfMethod.GetMethod(MetadataTokens.EntityHandle((int)methodToken));
MethodDesc methodOnType = methodType.FindMethodOnTypeWithMatchingTypicalMethod(ecmaMethod);
MethodDesc methodFound = methodOnType;
if (Macros.IsInstantiationNeeded(flags))
{
int instantiationArgumentCount = methodOnType.Instantiation.Length;
Debug.Assert(instantiationArgumentCount > 0);
List <TypeDesc> instantiationArguments = new List <TypeDesc>();
for (int i = 0; i < instantiationArgumentCount; i++)
{
instantiationArguments.Add(GetSigTypeFromIBCZapSig(ibcModule, ibcModule.EcmaModule, sig));
SkipTypeInIBCZapSig(ref sig);
}
methodFound = methodOnType.MakeInstantiatedMethod(new Instantiation(instantiationArguments.ToArray()));
}
if (Macros.IsUnboxingStub(flags))
{
_logger.Writer.WriteLine($"Warning: Skipping IBC data for unboxing stub {methodFound}");
return(null);
}
if (Macros.IsInstantiatingStub(flags))
{
_logger.Writer.WriteLine($"Warning: Skipping IBC data for instantiating stub {methodFound}");
return(null);
}
return(methodFound);
}
}
private EcmaModule AddModule(string filePath, string expectedSimpleName, byte[] moduleDataBytes, bool useForBinding, bool throwIfNotLoadable = true)
{
MemoryMappedViewAccessor mappedViewAccessor = null;
PdbSymbolReader pdbReader = null;
try
{
PEReader peReader = OpenPEFile(filePath, moduleDataBytes, out mappedViewAccessor);
if ((!peReader.HasMetadata) && !throwIfNotLoadable)
{
return(null);
}
pdbReader = OpenAssociatedSymbolFile(filePath, peReader);
EcmaModule module = EcmaModule.Create(this, peReader, containingAssembly: null, pdbReader);
MetadataReader metadataReader = module.MetadataReader;
string simpleName = metadataReader.GetString(metadataReader.GetAssemblyDefinition().Name);
ModuleData moduleData = new ModuleData()
{
SimpleName = simpleName,
FilePath = filePath,
Module = module,
MappedViewAccessor = mappedViewAccessor
};
lock (this)
{
if (useForBinding)
{
ModuleData actualModuleData;
if (!_simpleNameHashtable.TryGetValue(moduleData.SimpleName, out actualModuleData))
{
_simpleNameHashtable.Add(moduleData.SimpleName, moduleData);
actualModuleData = moduleData;
}
if (actualModuleData != moduleData)
{
if (actualModuleData.FilePath != filePath)
{
throw new FileNotFoundException("Module with same simple name already exists " + filePath);
}
return(actualModuleData.Module);
}
}
mappedViewAccessor = null; // Ownership has been transfered
pdbReader = null; // Ownership has been transferred
_moduleHashtable.AddOrGetExisting(moduleData);
}
return(module);
}
catch when(!throwIfNotLoadable)
{
return(null);
}
finally
{
if (mappedViewAccessor != null)
{
mappedViewAccessor.Dispose();
}
if (pdbReader != null)
{
pdbReader.Dispose();
}
}
}
public override ICompilation ToCompilation()
{
// TODO: only copy COR headers for single-assembly build and for composite build with embedded MSIL
IEnumerable <EcmaModule> inputModules = _compilationGroup.CompilationModuleSet;
EcmaModule singleModule = _compilationGroup.IsCompositeBuildMode ? null : inputModules.First();
CopiedCorHeaderNode corHeaderNode = new CopiedCorHeaderNode(singleModule);
// TODO: proper support for multiple input files
DebugDirectoryNode debugDirectoryNode = new DebugDirectoryNode(singleModule, _outputFile, _generatePdbFile, _generatePerfMapFile);
// Produce a ResourceData where the IBC PROFILE_DATA entry has been filtered out
// TODO: proper support for multiple input files
ResourceData win32Resources = new ResourceData(inputModules.First(), (object type, object name, ushort language) =>
{
if (!(type is string) || !(name is string))
{
return(true);
}
if (language != 0)
{
return(true);
}
string typeString = (string)type;
string nameString = (string)name;
if ((typeString == "IBC") && (nameString == "PROFILE_DATA"))
{
return(false);
}
return(true);
});
ReadyToRunFlags flags = ReadyToRunFlags.READYTORUN_FLAG_NonSharedPInvokeStubs;
if (inputModules.All(module => module.IsPlatformNeutral))
{
flags |= ReadyToRunFlags.READYTORUN_FLAG_PlatformNeutralSource;
}
flags |= _compilationGroup.GetReadyToRunFlags();
NodeFactory factory = new NodeFactory(
_context,
(ReadyToRunCompilationModuleGroupBase)_compilationGroup,
_profileData,
_nameMangler,
corHeaderNode,
debugDirectoryNode,
win32Resources,
flags);
factory.CompositeImageSettings = _compositeImageSettings;
IComparer <DependencyNodeCore <NodeFactory> > comparer = new SortableDependencyNode.ObjectNodeComparer(CompilerComparer.Instance);
DependencyAnalyzerBase <NodeFactory> graph = CreateDependencyGraph(factory, comparer);
List <CorJitFlag> corJitFlags = new List <CorJitFlag> {
CorJitFlag.CORJIT_FLAG_DEBUG_INFO
};
switch (_optimizationMode)
{
case OptimizationMode.None:
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_DEBUG_CODE);
break;
case OptimizationMode.PreferSize:
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_SIZE_OPT);
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_BBOPT);
break;
case OptimizationMode.PreferSpeed:
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_SPEED_OPT);
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_BBOPT);
break;
default:
// Not setting a flag results in BLENDED_CODE.
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_BBOPT);
break;
}
if (_ibcTuning)
{
corJitFlags.Add(CorJitFlag.CORJIT_FLAG_BBINSTR);
}
if (!_isJitInitialized)
{
JitConfigProvider.Initialize(_context.Target, corJitFlags, _ryujitOptions, _jitPath);
_isJitInitialized = true;
}
return(new ReadyToRunCodegenCompilation(
graph,
factory,
_compilationRoots,
_ilProvider,
_logger,
new DependencyAnalysis.ReadyToRun.DevirtualizationManager(_compilationGroup),
_inputFiles,
_compositeRootPath,
_instructionSetSupport,
resilient: _resilient,
generateMapFile: _generateMapFile,
generateMapCsvFile: _generateMapCsvFile,
generatePdbFile: _generatePdbFile,
printReproInstructions: _printReproInstructions,
pdbPath: _pdbPath,
generatePerfMapFile: _generatePerfMapFile,
perfMapPath: _perfMapPath,
perfMapFormatVersion: _perfMapFormatVersion,
generateProfileFile: _generateProfileFile,
_parallelism,
_profileData,
_r2rMethodLayoutAlgorithm,
_r2rFileLayoutAlgorithm,
_customPESectionAlignment,
_verifyTypeAndFieldLayout));
}
public TypeDesc ResolveTypeHandle(long handle, bool throwIfNotFound = true)
{
lock (_lock)
{
TypeHandleInfo tinfo;
if (_types.TryGetValue(handle, out tinfo))
{
if (tinfo.Type != null)
{
return(tinfo.Type);
}
if ((tinfo.TypeValue.Flags & Microsoft.Diagnostics.Tracing.Parsers.Clr.TypeFlags.Array) != 0)
{
if (tinfo.TypeValue.TypeParameters.Length != 1)
{
throw new Exception("Bad format for BulkType");
}
TypeDesc elementType = ResolveTypeHandle((long)tinfo.TypeValue.TypeParameters[0], throwIfNotFound);
if (elementType == null)
{
return(null);
}
if (tinfo.TypeValue.CorElementType == (byte)SignatureTypeCode.SZArray)
{
tinfo.Type = elementType.MakeArrayType();
}
else
{
int rank = tinfo.TypeValue.Flags.GetArrayRank();
tinfo.Type = elementType.MakeArrayType(rank);
}
}
else if (tinfo.TypeValue.CorElementType == (byte)SignatureTypeCode.ByReference)
{
if (tinfo.TypeValue.TypeParameters.Length != 1)
{
throw new Exception("Bad format for BulkType");
}
TypeDesc elementType = ResolveTypeHandle((long)tinfo.TypeValue.TypeParameters[0], throwIfNotFound);
if (elementType == null)
{
return(null);
}
tinfo.Type = elementType.MakeByRefType();
}
else if (tinfo.TypeValue.CorElementType == (byte)SignatureTypeCode.Pointer)
{
if (tinfo.TypeValue.TypeParameters.Length != 1)
{
throw new Exception("Bad format for BulkType");
}
TypeDesc elementType = ResolveTypeHandle((long)tinfo.TypeValue.TypeParameters[0], throwIfNotFound);
if (elementType == null)
{
return(null);
}
tinfo.Type = elementType.MakePointerType();
}
else if (tinfo.TypeValue.CorElementType == (byte)SignatureTypeCode.FunctionPointer)
{
tinfo.Type = null;
}
else
{
// Must be class type or instantiated type.
ModuleDesc module = ResolveModuleID((long)tinfo.TypeValue.ModuleID, throwIfNotFound);
if (module == null)
{
return(null);
}
EcmaModule ecmaModule = module as EcmaModule;
if (ecmaModule == null)
{
if (throwIfNotFound)
{
throw new Exception($"Unable to resolve module for {handle:8x}");
}
return(null);
}
if ((tinfo.TypeValue.TypeNameID & 0xFF000000) != 0x02000000)
{
throw new Exception($"Invalid typedef {tinfo.TypeValue.TypeNameID:4x}");
}
TypeDefinitionHandle typedef = MetadataTokens.TypeDefinitionHandle(tinfo.TypeValue.TypeNameID & 0xFFFFFF);
MetadataType uninstantiatedType = (MetadataType)ecmaModule.GetType(typedef);
// Instantiate the type if requested
if ((tinfo.TypeValue.TypeParameters.Length != 0) && uninstantiatedType != null)
{
if (uninstantiatedType.Instantiation.Length != tinfo.TypeValue.TypeParameters.Length)
{
throw new Exception($"Invalid TypeParameterCount {tinfo.TypeValue.TypeParameters.Length} expected {uninstantiatedType.Instantiation.Length} as needed by '{uninstantiatedType}'");
}
TypeDesc[] instantiation = new TypeDesc[tinfo.TypeValue.TypeParameters.Length];
for (int i = 0; i < instantiation.Length; i++)
{
instantiation[i] = ResolveTypeHandle((long)tinfo.TypeValue.TypeParameters[i], throwIfNotFound);
if (instantiation[i] == null)
{
return(null);
}
}
tinfo.Type = uninstantiatedType.Context.GetInstantiatedType(uninstantiatedType, new Instantiation(instantiation));
}
else
{
if ((uninstantiatedType.Name == "__Canon") && uninstantiatedType.Namespace == "System" && (uninstantiatedType.Module == uninstantiatedType.Context.SystemModule))
{
tinfo.Type = uninstantiatedType.Context.CanonType;
}
else
{
tinfo.Type = uninstantiatedType;
}
}
}
if (tinfo.Type == null)
{
if (throwIfNotFound)
{
throw new Exception("Unknown typeHandle value");
}
return(null);
}
return(tinfo.Type);
}
else
{
if (throwIfNotFound)
{
throw new Exception("Unknown typeHandle value");
}
return(null);
}
}
}
private int Run(string[] args)
{
InitializeDefaultOptions();
ArgumentSyntax syntax = ParseCommandLine(args);
if (_help)
{
Help(syntax.GetHelpText());
return(1);
}
if (_outputFilePath == null)
{
throw new CommandLineException("Output filename must be specified (/out <file>)");
}
//
// Set target Architecture and OS
//
if (_targetArchitectureStr != null)
{
if (_targetArchitectureStr.Equals("x86", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X86;
}
else if (_targetArchitectureStr.Equals("x64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X64;
}
else if (_targetArchitectureStr.Equals("arm", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM;
}
else if (_targetArchitectureStr.Equals("armel", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM;
}
else if (_targetArchitectureStr.Equals("arm64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM64;
}
else if (_targetArchitectureStr.Equals("wasm", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.Wasm32;
_isWasmCodegen = true;
}
else
{
throw new CommandLineException("Target architecture is not supported");
}
}
if (_targetOSStr != null)
{
if (_targetOSStr.Equals("windows", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Windows;
}
else if (_targetOSStr.Equals("linux", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Linux;
}
else if (_targetOSStr.Equals("osx", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.OSX;
}
else
{
throw new CommandLineException("Target OS is not supported");
}
}
if (_isWasmCodegen)
{
_targetArchitecture = TargetArchitecture.Wasm32;
}
//
// Initialize type system context
//
SharedGenericsMode genericsMode = _useSharedGenerics || !_isWasmCodegen ?
SharedGenericsMode.CanonicalReferenceTypes : SharedGenericsMode.Disabled;
// TODO: compiler switch for SIMD support?
var simdVectorLength = (_isCppCodegen || _isWasmCodegen) ? SimdVectorLength.None : SimdVectorLength.Vector128Bit;
var targetAbi = _isCppCodegen ? TargetAbi.CppCodegen : TargetAbi.CoreRT;
var targetDetails = new TargetDetails(_targetArchitecture, _targetOS, targetAbi, simdVectorLength);
CompilerTypeSystemContext typeSystemContext = (_isReadyToRunCodeGen
? new ReadyToRunCompilerContext(targetDetails, genericsMode)
: new CompilerTypeSystemContext(targetDetails, genericsMode));
//
// TODO: To support our pre-compiled test tree, allow input files that aren't managed assemblies since
// some tests contain a mixture of both managed and native binaries.
//
// See: https://github.com/dotnet/corert/issues/2785
//
// When we undo this this hack, replace this foreach with
// typeSystemContext.InputFilePaths = _inputFilePaths;
//
Dictionary <string, string> inputFilePaths = new Dictionary <string, string>();
foreach (var inputFile in _inputFilePaths)
{
try
{
var module = typeSystemContext.GetModuleFromPath(inputFile.Value);
inputFilePaths.Add(inputFile.Key, inputFile.Value);
}
catch (TypeSystemException.BadImageFormatException)
{
// Keep calm and carry on.
}
}
typeSystemContext.InputFilePaths = inputFilePaths;
typeSystemContext.ReferenceFilePaths = _referenceFilePaths;
typeSystemContext.SetSystemModule(typeSystemContext.GetModuleForSimpleName(_systemModuleName));
if (typeSystemContext.InputFilePaths.Count == 0)
{
throw new CommandLineException("No input files specified");
}
//
// Initialize compilation group and compilation roots
//
// Single method mode?
MethodDesc singleMethod = CheckAndParseSingleMethodModeArguments(typeSystemContext);
CompilationModuleGroup compilationGroup;
List <ICompilationRootProvider> compilationRoots = new List <ICompilationRootProvider>();
if (singleMethod != null)
{
// Compiling just a single method
compilationGroup = new SingleMethodCompilationModuleGroup(singleMethod);
compilationRoots.Add(new SingleMethodRootProvider(singleMethod));
}
else
{
// Either single file, or multifile library, or multifile consumption.
EcmaModule entrypointModule = null;
bool systemModuleIsInputModule = false;
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
if (module.PEReader.PEHeaders.IsExe)
{
if (entrypointModule != null)
{
throw new Exception("Multiple EXE modules");
}
entrypointModule = module;
}
if (module == typeSystemContext.SystemModule)
{
systemModuleIsInputModule = true;
}
if (!_isReadyToRunCodeGen)
{
compilationRoots.Add(new ExportedMethodsRootProvider(module));
}
}
if (entrypointModule != null && !_isReadyToRunCodeGen)
{
compilationRoots.Add(new MainMethodRootProvider(entrypointModule, CreateInitializerList(typeSystemContext)));
compilationRoots.Add(new RuntimeConfigurationRootProvider(_runtimeOptions));
}
if (_isReadyToRunCodeGen)
{
List <EcmaModule> inputModules = new List <EcmaModule>();
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
compilationRoots.Add(new ReadyToRunRootProvider(module));
inputModules.Add(module);
if (!_isInputVersionBubble)
{
break;
}
}
compilationGroup = new ReadyToRunSingleAssemblyCompilationModuleGroup(typeSystemContext, inputModules);
}
else if (_multiFile)
{
List <EcmaModule> inputModules = new List <EcmaModule>();
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
if (entrypointModule == null)
{
// This is a multifile production build - we need to root all methods
compilationRoots.Add(new LibraryRootProvider(module));
}
inputModules.Add(module);
}
compilationGroup = new MultiFileSharedCompilationModuleGroup(typeSystemContext, inputModules);
}
else
{
if (entrypointModule == null && !_nativeLib)
{
throw new Exception("No entrypoint module");
}
if (!systemModuleIsInputModule)
{
compilationRoots.Add(new ExportedMethodsRootProvider((EcmaModule)typeSystemContext.SystemModule));
}
compilationGroup = new SingleFileCompilationModuleGroup();
if (_rootAllApplicationAssemblies)
{
compilationRoots.Add(new ApplicationAssemblyRootProvider(typeSystemContext));
}
}
if (_nativeLib)
{
// Set owning module of generated native library startup method to compiler generated module,
// to ensure the startup method is included in the object file during multimodule mode build
compilationRoots.Add(new NativeLibraryInitializerRootProvider(typeSystemContext.GeneratedAssembly, CreateInitializerList(typeSystemContext)));
compilationRoots.Add(new RuntimeConfigurationRootProvider(_runtimeOptions));
}
if (_rdXmlFilePaths.Count > 0)
{
Console.WriteLine("Warning: RD.XML processing will change before release (https://github.com/dotnet/corert/issues/5001)");
}
foreach (var rdXmlFilePath in _rdXmlFilePaths)
{
compilationRoots.Add(new RdXmlRootProvider(typeSystemContext, rdXmlFilePath));
}
}
//
// Compile
//
CompilationBuilder builder;
if (_isWasmCodegen)
{
builder = new WebAssemblyCodegenCompilationBuilder(typeSystemContext, compilationGroup);
}
else if (_isReadyToRunCodeGen)
{
string inputFilePath = "";
foreach (var input in typeSystemContext.InputFilePaths)
{
inputFilePath = input.Value;
break;
}
builder = new ReadyToRunCodegenCompilationBuilder(typeSystemContext, compilationGroup, inputFilePath);
}
else if (_isCppCodegen)
{
builder = new CppCodegenCompilationBuilder(typeSystemContext, compilationGroup);
}
else
{
builder = new RyuJitCompilationBuilder(typeSystemContext, compilationGroup);
}
string compilationUnitPrefix = _multiFile ? System.IO.Path.GetFileNameWithoutExtension(_outputFilePath) : "";
builder.UseCompilationUnitPrefix(compilationUnitPrefix);
if (!_isCppCodegen && !_isWasmCodegen)
{
builder.UsePInvokePolicy(new ConfigurablePInvokePolicy(typeSystemContext.Target));
}
RemovedFeature removedFeatures = 0;
foreach (string feature in _removedFeatures)
{
if (feature == "EventSource")
{
removedFeatures |= RemovedFeature.Etw;
}
else if (feature == "FrameworkStrings")
{
removedFeatures |= RemovedFeature.FrameworkResources;
}
else if (feature == "Globalization")
{
removedFeatures |= RemovedFeature.Globalization;
}
else if (feature == "Comparers")
{
removedFeatures |= RemovedFeature.Comparers;
}
else if (feature == "CurlHandler")
{
removedFeatures |= RemovedFeature.CurlHandler;
}
}
ILProvider ilProvider = _isReadyToRunCodeGen ? (ILProvider) new ReadyToRunILProvider() : new CoreRTILProvider();
if (removedFeatures != 0)
{
ilProvider = new RemovingILProvider(ilProvider, removedFeatures);
}
var stackTracePolicy = _emitStackTraceData ?
(StackTraceEmissionPolicy) new EcmaMethodStackTraceEmissionPolicy() : new NoStackTraceEmissionPolicy();
MetadataBlockingPolicy mdBlockingPolicy = _noMetadataBlocking
? (MetadataBlockingPolicy) new NoMetadataBlockingPolicy()
: new BlockedInternalsBlockingPolicy(typeSystemContext);
ManifestResourceBlockingPolicy resBlockingPolicy = (removedFeatures & RemovedFeature.FrameworkResources) != 0 ?
new FrameworkStringResourceBlockingPolicy() : (ManifestResourceBlockingPolicy) new NoManifestResourceBlockingPolicy();
UsageBasedMetadataGenerationOptions metadataGenerationOptions = UsageBasedMetadataGenerationOptions.AnonymousTypeHeuristic;
if (_completeTypesMetadata)
{
metadataGenerationOptions |= UsageBasedMetadataGenerationOptions.CompleteTypesOnly;
}
if (_scanReflection)
{
metadataGenerationOptions |= UsageBasedMetadataGenerationOptions.ILScanning;
}
DynamicInvokeThunkGenerationPolicy invokeThunkGenerationPolicy = new DefaultDynamicInvokeThunkGenerationPolicy();
bool supportsReflection = !_isReadyToRunCodeGen && _systemModuleName == DefaultSystemModule;
MetadataManager metadataManager;
if (_isReadyToRunCodeGen)
{
metadataManager = new ReadyToRunTableManager(typeSystemContext);
}
else if (supportsReflection)
{
metadataManager = new UsageBasedMetadataManager(
compilationGroup,
typeSystemContext,
mdBlockingPolicy,
resBlockingPolicy,
_metadataLogFileName,
stackTracePolicy,
invokeThunkGenerationPolicy,
ilProvider,
metadataGenerationOptions);
}
else
{
metadataManager = new EmptyMetadataManager(typeSystemContext);
}
// Unless explicitly opted in at the command line, we enable scanner for retail builds by default.
// We don't do this for CppCodegen and Wasm, because those codegens are behind.
// We also don't do this for multifile because scanner doesn't simulate inlining (this would be
// fixable by using a CompilationGroup for the scanner that has a bigger worldview, but
// let's cross that bridge when we get there).
bool useScanner = _useScanner ||
(_optimizationMode != OptimizationMode.None && !_isCppCodegen && !_isWasmCodegen && !_isReadyToRunCodeGen && !_multiFile);
useScanner &= !_noScanner;
builder.UseILProvider(ilProvider);
ILScanResults scanResults = null;
if (useScanner)
{
ILScannerBuilder scannerBuilder = builder.GetILScannerBuilder()
.UseCompilationRoots(compilationRoots)
.UseMetadataManager(metadataManager);
if (_scanDgmlLogFileName != null)
{
scannerBuilder.UseDependencyTracking(_generateFullScanDgmlLog ? DependencyTrackingLevel.All : DependencyTrackingLevel.First);
}
IILScanner scanner = scannerBuilder.ToILScanner();
scanResults = scanner.Scan();
if (metadataManager is UsageBasedMetadataManager usageBasedManager)
{
metadataManager = usageBasedManager.ToAnalysisBasedMetadataManager();
}
}
var logger = new Logger(Console.Out, _isVerbose);
DebugInformationProvider debugInfoProvider = _enableDebugInfo ?
(_ilDump == null ? new DebugInformationProvider() : new ILAssemblyGeneratingMethodDebugInfoProvider(_ilDump, new EcmaOnlyDebugInformationProvider())) :
new NullDebugInformationProvider();
DependencyTrackingLevel trackingLevel = _dgmlLogFileName == null ?
DependencyTrackingLevel.None : (_generateFullDgmlLog ? DependencyTrackingLevel.All : DependencyTrackingLevel.First);
compilationRoots.Add(metadataManager);
builder
.UseBackendOptions(_codegenOptions)
.UseMetadataManager(metadataManager)
.UseLogger(logger)
.UseDependencyTracking(trackingLevel)
.UseCompilationRoots(compilationRoots)
.UseOptimizationMode(_optimizationMode)
.UseDebugInfoProvider(debugInfoProvider);
if (scanResults != null)
{
// If we have a scanner, feed the vtable analysis results to the compilation.
// This could be a command line switch if we really wanted to.
builder.UseVTableSliceProvider(scanResults.GetVTableLayoutInfo());
// If we have a scanner, feed the generic dictionary results to the compilation.
// This could be a command line switch if we really wanted to.
builder.UseGenericDictionaryLayoutProvider(scanResults.GetDictionaryLayoutInfo());
// If we feed any outputs of the scanner into the compilation, it's essential
// we use scanner's devirtualization manager. It prevents optimizing codegens
// from accidentally devirtualizing cases that can never happen at runtime
// (e.g. devirtualizing a method on a type that never gets allocated).
builder.UseDevirtualizationManager(scanResults.GetDevirtualizationManager());
}
ICompilation compilation = builder.ToCompilation();
ObjectDumper dumper = _mapFileName != null ? new ObjectDumper(_mapFileName) : null;
CompilationResults compilationResults = compilation.Compile(_outputFilePath, dumper);
if (_exportsFile != null)
{
ExportsFileWriter defFileWriter = new ExportsFileWriter(typeSystemContext, _exportsFile);
foreach (var compilationRoot in compilationRoots)
{
if (compilationRoot is ExportedMethodsRootProvider provider)
{
defFileWriter.AddExportedMethods(provider.ExportedMethods);
}
}
defFileWriter.EmitExportedMethods();
}
if (_dgmlLogFileName != null)
{
compilationResults.WriteDependencyLog(_dgmlLogFileName);
}
if (scanResults != null)
{
SimdHelper simdHelper = new SimdHelper();
if (_scanDgmlLogFileName != null)
{
scanResults.WriteDependencyLog(_scanDgmlLogFileName);
}
// If the scanner and compiler don't agree on what to compile, the outputs of the scanner might not actually be usable.
// We are going to check this two ways:
// 1. The methods and types generated during compilation are a subset of method and types scanned
// 2. The methods and types scanned are a subset of methods and types compiled (this has a chance to hold for unoptimized builds only).
// Check that methods and types generated during compilation are a subset of method and types scanned
bool scanningFail = false;
DiffCompilationResults(ref scanningFail, compilationResults.CompiledMethodBodies, scanResults.CompiledMethodBodies,
"Methods", "compiled", "scanned", method => !(method.GetTypicalMethodDefinition() is EcmaMethod) || method.Name == "ThrowPlatformNotSupportedException");
DiffCompilationResults(ref scanningFail, compilationResults.ConstructedEETypes, scanResults.ConstructedEETypes,
"EETypes", "compiled", "scanned", type => !(type.GetTypeDefinition() is EcmaType));
// If optimizations are enabled, the results will for sure not match in the other direction due to inlining, etc.
// But there's at least some value in checking the scanner doesn't expand the universe too much in debug.
if (_optimizationMode == OptimizationMode.None)
{
// Check that methods and types scanned are a subset of methods and types compiled
// If we find diffs here, they're not critical, but still might be causing a Size on Disk regression.
bool dummy = false;
// We additionally skip methods in SIMD module because there's just too many intrisics to handle and IL scanner
// doesn't expand them. They would show up as noisy diffs.
DiffCompilationResults(ref dummy, scanResults.CompiledMethodBodies, compilationResults.CompiledMethodBodies,
"Methods", "scanned", "compiled", method => !(method.GetTypicalMethodDefinition() is EcmaMethod) || simdHelper.IsSimdType(method.OwningType));
DiffCompilationResults(ref dummy, scanResults.ConstructedEETypes, compilationResults.ConstructedEETypes,
"EETypes", "scanned", "compiled", type => !(type.GetTypeDefinition() is EcmaType));
}
if (scanningFail)
{
throw new Exception("Scanning failure");
}
}
if (debugInfoProvider is IDisposable)
{
((IDisposable)debugInfoProvider).Dispose();
}
return(0);
}
public ExportedMethodsRootProvider(EcmaModule module)
{
_module = module;
}
public IEnumerable <IMethodNode> GetCompiledMethods(EcmaModule moduleToEnumerate, CompiledMethodCategory methodCategory)
{
lock (_methodsGenerated)
{
if (!_sortedMethods)
{
TypeSystemComparer comparer = new TypeSystemComparer();
Comparison <IMethodNode> sortHelper = (x, y) => comparer.Compare(x.Method, y.Method);
List <PerModuleMethodsGenerated> perModuleDatas = new List <PerModuleMethodsGenerated>(_methodsGenerated.Values);
perModuleDatas.Sort((x, y) => x.Module.CompareTo(y.Module));
foreach (var perModuleData in perModuleDatas)
{
perModuleData.MethodsGenerated.MergeSort(sortHelper);
perModuleData.GenericMethodsGenerated.MergeSort(sortHelper);
_completeSortedMethods.AddRange(perModuleData.MethodsGenerated);
_completeSortedMethods.AddRange(perModuleData.GenericMethodsGenerated);
_completeSortedGenericMethods.AddRange(perModuleData.GenericMethodsGenerated);
}
_completeSortedMethods.MergeSort(sortHelper);
_completeSortedGenericMethods.MergeSort(sortHelper);
_sortedMethods = true;
}
}
if (moduleToEnumerate == null)
{
if (methodCategory == CompiledMethodCategory.All)
{
return(_completeSortedMethods);
}
else if (methodCategory == CompiledMethodCategory.Instantiated)
{
return(_completeSortedGenericMethods);
}
else
{
// This isn't expected to be needed, and thus isn't implemented
throw new ArgumentException();
}
}
else if (_methodsGenerated.TryGetValue(moduleToEnumerate, out var perModuleData))
{
if (methodCategory == CompiledMethodCategory.All)
{
return(GetCompiledMethodsAllMethodsInModuleHelper(moduleToEnumerate));
}
if (methodCategory == CompiledMethodCategory.Instantiated)
{
return(perModuleData.GenericMethodsGenerated);
}
else
{
Debug.Assert(methodCategory == CompiledMethodCategory.NonInstantiated);
return(perModuleData.MethodsGenerated);
}
}
else
{
return(Array.Empty <IMethodNode>());
}
}
public PerModuleMethodsGenerated(EcmaModule module)
{
Module = module;
}
public CopiedManagedResourcesNode CopiedManagedResources(EcmaModule module)
{
return(_copiedManagedResources.GetOrAdd(module));
}
/// <summary>
/// Parse an MIBC file for the methods that are interesting.
/// The version bubble must be specified and will describe the restrict the set of methods parsed to those relevant to the compilation
/// The onlyDefinedInAssembly parameter is used to restrict the set of types parsed to include only those which are defined in a specific module. Specify null to allow definitions from all modules.
/// This limited parsing is not necessarily an exact set of prevention, so detailed algorithms that work at the individual method level are still necessary, but this allows avoiding excessive parsing.
///
/// The format of the Mibc file is that of a .NET dll, with a global method named "AssemblyDictionary". Inside of that file are a series of references that are broken up by which assemblies define the individual methods.
/// These references are encoded as IL code that represents the details.
/// The format of these IL instruction is as follows.
///
/// ldstr mibcGroupName
/// ldtoken mibcGroupMethod
/// pop
/// {Repeat the above pattern N times, once per Mibc group}
///
/// See comment above ReadMIbcGroup for details of the group format
///
/// The mibcGroupName is in the following format "Assembly_{definingAssemblyName};{OtherAssemblyName};{OtherAssemblyName};...; (OtherAssemblyName is ; delimited)
///
/// </summary>
/// <returns></returns>
public static ProfileData ParseMIbcFile(TypeSystemContext tsc, PEReader peReader, HashSet <string> assemblyNamesInVersionBubble, string onlyDefinedInAssembly)
{
var mibcModule = EcmaModule.Create(tsc, peReader, null, null, new CustomCanonResolver(tsc));
var assemblyDictionary = (EcmaMethod)mibcModule.GetGlobalModuleType().GetMethod("AssemblyDictionary", null);
IEnumerable <MethodProfileData> loadedMethodProfileData = Enumerable.Empty <MethodProfileData>();
EcmaMethodIL ilBody = EcmaMethodIL.Create(assemblyDictionary);
ILReader ilReader = new ILReader(ilBody.GetILBytes());
string mibcGroupName = "";
while (ilReader.HasNext)
{
ILOpcode opcode = ilReader.ReadILOpcode();
switch (opcode)
{
case ILOpcode.ldstr:
int userStringToken = ilReader.ReadILToken();
Debug.Assert(mibcGroupName == "");
if (mibcGroupName == "")
{
mibcGroupName = (string)ilBody.GetObject(userStringToken);
}
break;
case ILOpcode.ldtoken:
int token = ilReader.ReadILToken();
if (String.IsNullOrEmpty(mibcGroupName))
{
break;
}
string[] assembliesByName = mibcGroupName.Split(';');
bool hasMatchingDefinition = (onlyDefinedInAssembly == null) || assembliesByName[0].Equals(onlyDefinedInAssembly);
if (!hasMatchingDefinition)
{
break;
}
if (assemblyNamesInVersionBubble != null)
{
bool areAllEntriesInVersionBubble = true;
foreach (string s in assembliesByName)
{
if (string.IsNullOrEmpty(s))
{
continue;
}
if (!assemblyNamesInVersionBubble.Contains(s))
{
areAllEntriesInVersionBubble = false;
break;
}
}
if (!areAllEntriesInVersionBubble)
{
break;
}
}
loadedMethodProfileData = loadedMethodProfileData.Concat(ReadMIbcGroup(tsc, (EcmaMethod)ilBody.GetObject(token)));
break;
case ILOpcode.pop:
mibcGroupName = "";
break;
default:
ilReader.Skip(opcode);
break;
}
}
return(new IBCProfileData(false, loadedMethodProfileData));
}
public ProfileData ParseIBCDataFromModule(EcmaModule ecmaModule)
{
ResourceData peResources = new ResourceData(ecmaModule);
byte[] ibcDataSection = peResources.FindResource("PROFILE_DATA", "IBC", 0);
if (ibcDataSection == null)
{
// If we don't have profile data, return empty ProfileData object
return(EmptyProfileData.Singleton);
}
var reader = new IBCDataReader();
int pos = 0;
bool basicBlocksOnly = false;
AssemblyData parsedData = reader.Read(ibcDataSection, ref pos, out bool minified);
if (parsedData.FormatMajorVersion == 1)
{
throw new Exception("Unsupported V1 IBC Format");
}
Dictionary <IBCBlobKey, BlobEntry> blobs = GetIBCBlobs(parsedData.BlobStream, out HashSet <uint> ignoredIbcMethodSpecTokens);
List <MethodProfileData> methodProfileData = new List <MethodProfileData>();
IBCModule ibcModule = new IBCModule(ecmaModule, blobs);
// Parse the token lists
IBCData.SectionIteratorKind iteratorKind = basicBlocksOnly ? IBCData.SectionIteratorKind.BasicBlocks : IBCData.SectionIteratorKind.TokenFlags;
foreach (SectionFormat section in IBCData.SectionIterator(iteratorKind))
{
if (!parsedData.Tokens.TryGetValue(section, out List <TokenData> TokenList) ||
TokenList.Count == 0)
{
continue;
}
// In V1 and minified V3+ files, tokens aren't stored with a
// scenario mask. In minified V3+ files, it can be treated as
// anything nonzero--minified files make no guarantee about
// preserving scenario information, but the flags must be left
// alone.
const uint scenarioMaskIfMissing = 1u;
foreach (TokenData entry in TokenList)
{
//
// Discard any token list entries which refer to the ParamMethodSpec blob stream entries
// (if any) which were thrown away above. Note that the MethodProfilingData token list is
// the only place anywhere in the IBC data which can ever contain an embedded ibcMethodSpec
// token.
//
if (section == SectionFormat.MethodProfilingData)
{
if (ignoredIbcMethodSpecTokens.Contains(entry.Token))
{
continue;
}
}
uint scenarioMask = entry.ScenarioMask ?? scenarioMaskIfMissing;
// scenarioMask will be 0 in unprocessed or V1 IBC data.
if (scenarioMask == 0)
{
throw new NotImplementedException();
/* Debug.Assert(fullScenarioMask == 1, "Token entry not owned by one scenario");
* // We have to compute the RunOnceMethod and RunNeverMethod flags.
* entry.Flags = result.GetFlags(entry.Flags, section, entry.Token);
* scenarioMask = defaultScenarioMask;*/
}
// Debug.Assert(((~fullScenarioMask & scenarioMask) == 0), "Illegal scenarios mask");
MethodDesc associatedMethod = null;
switch (Cor.Macros.TypeFromToken(entry.Token))
{
case CorTokenType.mdtMethodDef:
case CorTokenType.mdtMemberRef:
case CorTokenType.mdtMethodSpec:
associatedMethod = ecmaModule.GetMethod(System.Reflection.Metadata.Ecma335.MetadataTokens.EntityHandle((int)entry.Token));
break;
case CorTokenType.ibcMethodSpec:
{
if (!blobs.TryGetValue(new IBCBlobKey(entry.Token, BlobType.ParamMethodSpec), out BlobEntry blobEntry))
{
throw new Exception($"Missing blob entry for ibcMethodSpec {entry.Token:x}");
}
BlobEntry.SignatureEntry paramSignatureEntry = blobEntry as BlobEntry.SignatureEntry;
if (paramSignatureEntry == null)
{
throw new Exception($"Blob entry for {entry.Token:x} is invalid");
}
unsafe
{
fixed(byte *pb = ¶mSignatureEntry.Signature[0])
{
BlobReader br = new BlobReader(pb, paramSignatureEntry.Signature.Length);
associatedMethod = GetSigMethodInstantiationFromIBCMethodSpec(ibcModule, br);
}
}
}
break;
}
if (associatedMethod != null)
{
methodProfileData.Add(new MethodProfileData(associatedMethod, (MethodProfilingDataFlags)entry.Flags, scenarioMask));
}
}
}
return(new IBCProfileData(parsedData.PartialNGen, methodProfileData));
}
// LookupIbcTypeToken and (possibly) find the module associated with it.
// externalModule may be null if the exact assembly isn't known
private uint LookupIbcTypeToken(ref EcmaModule externalModule, uint ibcToken, Dictionary <IBCBlobKey, BlobEntry> blobs)
{
if (!blobs.TryGetValue(new IBCBlobKey(ibcToken, BlobType.ExternalTypeDef), out BlobEntry externalTypeDefBlob))
{
if (_logger.IsVerbose)
{
_logger.Writer.WriteLine($"Ibc TypeToken {ibcToken:x} unable to find external typedef");
}
return(Cor.Macros.RidToToken(0, CorTokenType.mdtTypeDef)); // Nil TypeDef token
}
var typeEntry = (BlobEntry.ExternalTypeEntry)externalTypeDefBlob;
string typeNamespace = "";
string typeName = Encoding.UTF8.GetString(typeEntry.Name, 0, typeEntry.Name.Length - 1 /* these strings are null terminated */);
TypeDefinitionHandle enclosingType = default;
if (!Cor.Macros.IsNilToken(typeEntry.NamespaceToken))
{
if (!Cor.Macros.IsNilToken(typeEntry.NestedClassToken))
{
// Do not support typedef with namespace that is nested
throw new Exception($"Ibc TypeToken {ibcToken:x} has both Namespace and NestedClass tokens");
}
uint nameSpaceToken = typeEntry.NamespaceToken;
if (Cor.Macros.TypeFromToken(nameSpaceToken) != CorTokenType.ibcExternalNamespace)
{
throw new Exception($"Ibc TypeToken {ibcToken:x} has Namespace tokens that is not a ibcExternalNamespace");
}
if (!blobs.TryGetValue(new IBCBlobKey(nameSpaceToken, BlobType.ExternalNamespaceDef), out BlobEntry namespaceEntryBlob))
{
if (_logger.IsVerbose)
{
_logger.Writer.WriteLine($"Ibc TypeToken {ibcToken:x} unable to find external namespace blob '{nameSpaceToken:x}");
}
return(Cor.Macros.RidToToken(0, CorTokenType.mdtTypeDef)); // Nil TypeDef token
}
var namespaceEntry = (BlobEntry.ExternalNamespaceEntry)namespaceEntryBlob;
typeNamespace = Encoding.UTF8.GetString(namespaceEntry.Name, 0, namespaceEntry.Name.Length - 1 /* these strings are null terminated */);
}
else if (!Cor.Macros.IsNilToken(typeEntry.NestedClassToken))
{
uint enclosingTypeTokenValue = LookupIbcTypeToken(ref externalModule, typeEntry.NestedClassToken, blobs);
if (Cor.Macros.TypeFromToken(enclosingTypeTokenValue) != CorTokenType.mdtTypeDef)
{
throw new Exception($"Ibc TypeToken {ibcToken:x} has NestedClass token which does not resolve to a type definition");
}
enclosingType = MetadataTokens.TypeDefinitionHandle((int)Cor.Macros.RidFromToken(enclosingTypeTokenValue));
if (enclosingType.IsNil && _logger.IsVerbose)
{
_logger.Writer.WriteLine($"Ibc TypeToken {ibcToken:x} has NestedClass token which resolves to a nil token");
}
}
if (enclosingType.IsNil)
{
EcmaType foundType = null;
if (externalModule == null)
{
// Lookup actual module scenario.
foreach (ModuleDesc m in _possibleReferenceModules)
{
if (!(m is EcmaModule))
{
continue;
}
foundType = (EcmaType)m.GetType(typeNamespace, typeName, throwIfNotFound: false);
if (foundType != null)
{
externalModule = foundType.EcmaModule;
break;
}
}
}
else
{
foundType = (EcmaType)externalModule.GetType(typeNamespace, typeName, throwIfNotFound: false);
}
if (foundType == null)
{
if (_logger.IsVerbose)
{
_logger.Writer.WriteLine($"Ibc TypeToken {ibcToken:x} has type token which resolves to a nil token");
}
return(Cor.Macros.RidToToken(0, CorTokenType.mdtTypeDef)); // Nil TypeDef token
}
return((uint)externalModule.MetadataReader.GetToken(foundType.Handle));
}
else
{
TypeDefinition nestedClassDefinition = externalModule.MetadataReader.GetTypeDefinition(enclosingType);
MetadataStringComparer stringComparer = externalModule.MetadataReader.StringComparer;
foreach (TypeDefinitionHandle tdNested in nestedClassDefinition.GetNestedTypes())
{
TypeDefinition candidateClassDefinition = externalModule.MetadataReader.GetTypeDefinition(tdNested);
if (stringComparer.Equals(candidateClassDefinition.Name, typeName))
{
return((uint)externalModule.MetadataReader.GetToken(tdNested));
}
}
if (_logger.IsVerbose)
{
_logger.Writer.WriteLine($"Ibc TypeToken {ibcToken:x} unable to find nested type '{typeName}' on type '{externalModule.MetadataReader.GetToken(enclosingType):x}'");
}
return(Cor.Macros.RidToToken(0, CorTokenType.mdtTypeDef)); // Nil TypeDef token
}
}
// Load type from IBC ZapSig. Returns null for cases where the type is legally defined, but is not used in R2R image generation
private TypeDesc GetSigTypeFromIBCZapSig(IBCModule ibcModule, EcmaModule ecmaModule, BlobReader sig)
{
TypeSystemContext context = ibcModule.EcmaModule.Context;
CorElementType typ = (CorElementType)sig.ReadByte();
switch (typ)
{
case CorElementType.ELEMENT_TYPE_VOID:
return(context.GetWellKnownType(WellKnownType.Void));
case CorElementType.ELEMENT_TYPE_BOOLEAN:
return(context.GetWellKnownType(WellKnownType.Boolean));
case CorElementType.ELEMENT_TYPE_CHAR:
return(context.GetWellKnownType(WellKnownType.Char));
case CorElementType.ELEMENT_TYPE_I1:
return(context.GetWellKnownType(WellKnownType.SByte));
case CorElementType.ELEMENT_TYPE_U1:
return(context.GetWellKnownType(WellKnownType.Byte));
case CorElementType.ELEMENT_TYPE_I2:
return(context.GetWellKnownType(WellKnownType.Int16));
case CorElementType.ELEMENT_TYPE_U2:
return(context.GetWellKnownType(WellKnownType.UInt16));
case CorElementType.ELEMENT_TYPE_I4:
return(context.GetWellKnownType(WellKnownType.Int32));
case CorElementType.ELEMENT_TYPE_U4:
return(context.GetWellKnownType(WellKnownType.UInt32));
case CorElementType.ELEMENT_TYPE_I8:
return(context.GetWellKnownType(WellKnownType.Int64));
case CorElementType.ELEMENT_TYPE_U8:
return(context.GetWellKnownType(WellKnownType.UInt64));
case CorElementType.ELEMENT_TYPE_R4:
return(context.GetWellKnownType(WellKnownType.Single));
case CorElementType.ELEMENT_TYPE_R8:
return(context.GetWellKnownType(WellKnownType.Double));
case CorElementType.ELEMENT_TYPE_STRING:
return(context.GetWellKnownType(WellKnownType.String));
case CorElementType.ELEMENT_TYPE_OBJECT:
return(context.GetWellKnownType(WellKnownType.Object));
case CorElementType.ELEMENT_TYPE_TYPEDBYREF:
return(context.GetWellKnownType(WellKnownType.TypedReference));
case CorElementType.ELEMENT_TYPE_CANON_ZAPSIG:
if (!context.SupportsCanon)
{
return(null);
}
return(context.CanonType);
case CorElementType.ELEMENT_TYPE_MODULE_ZAPSIG:
// Check version bubble by looking at reference to non-local module
// If null, then the remote reference is not permitted.
EcmaModule remoteModule = ibcModule.GetModuleFromIndex(sig.ReadCompressedInteger());
return(GetSigTypeFromIBCZapSig(ibcModule, remoteModule, sig));
case CorElementType.ELEMENT_TYPE_VAR:
case CorElementType.ELEMENT_TYPE_MVAR:
// VAR/MVAR can never appear in a ZapSig
throw new Exception("Attempt to parse ELEMENT_TYPE_VAR or ELEMENT_TYPE_MVAR in an IBC ZapSig");
case CorElementType.ELEMENT_TYPE_GENERICINST:
CorElementType genericTyp = (CorElementType)sig.ReadByte();
MetadataType genericDefinitionType = LoadTypeFromIBCZapSig(ibcModule, ecmaModule, genericTyp, ref sig);
if (genericDefinitionType == null)
{
return(null);
}
int numTypeArgs = sig.ReadCompressedInteger();
TypeDesc[] typeArgs = new TypeDesc[numTypeArgs];
for (int i = 0; i < numTypeArgs; i++)
{
TypeDesc nextTypeArg = GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig);
if (nextTypeArg == null)
{
return(null);
}
SkipTypeInIBCZapSig(ref sig);
typeArgs[i] = nextTypeArg;
}
return(genericDefinitionType.MakeInstantiatedType(new Instantiation(typeArgs)));
case CorElementType.ELEMENT_TYPE_CLASS:
case CorElementType.ELEMENT_TYPE_VALUETYPE:
return(LoadTypeFromIBCZapSig(ibcModule, ecmaModule, typ, ref sig));
case CorElementType.ELEMENT_TYPE_SZARRAY:
{
TypeDesc arrayElementType = GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig);
if (arrayElementType == null)
{
return(null);
}
return(arrayElementType.MakeArrayType());
}
case CorElementType.ELEMENT_TYPE_ARRAY:
{
TypeDesc arrayElementType = GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig);
if (arrayElementType == null)
{
return(null);
}
SkipTypeInIBCZapSig(ref sig);
return(arrayElementType.MakeArrayType(sig.ReadCompressedInteger()));
}
case CorElementType.ELEMENT_TYPE_PINNED:
// Return what follows
return(GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig));
case CorElementType.ELEMENT_TYPE_BYREF:
{
TypeDesc byRefToType = GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig);
if (byRefToType == null)
{
return(null);
}
return(byRefToType.MakeByRefType());
}
case CorElementType.ELEMENT_TYPE_PTR:
{
TypeDesc pointerToType = GetSigTypeFromIBCZapSig(ibcModule, ecmaModule, sig);
if (pointerToType == null)
{
return(null);
}
return(pointerToType.MakePointerType());
}
default:
throw new Exception($"Invalid element type {typ:x} in IBC ZapSig");
}
}
public ModuleAndIntValueKey(int integer, EcmaModule module)
{
IntValue = integer;
Module = module;
}
public CopiedCorHeaderNode(EcmaModule sourceModule)
{
_module = sourceModule;
}
public CopiedCorHeaderNode CopiedCorHeader(EcmaModule module)
{
return(_copiedCorHeaders.GetOrAdd(module));
}
private EcmaModule AddModule(string filePath, string expectedSimpleName, bool useForBinding)
{
MemoryMappedViewAccessor mappedViewAccessor = null;
PdbSymbolReader pdbReader = null;
try
{
PEReader peReader = OpenPEFile(filePath, out mappedViewAccessor);
pdbReader = OpenAssociatedSymbolFile(filePath, peReader);
EcmaModule module = EcmaModule.Create(this, peReader, pdbReader);
MetadataReader metadataReader = module.MetadataReader;
string simpleName = metadataReader.GetString(metadataReader.GetAssemblyDefinition().Name);
if (expectedSimpleName != null && !simpleName.Equals(expectedSimpleName, StringComparison.OrdinalIgnoreCase))
{
throw new FileNotFoundException("Assembly name does not match filename " + filePath);
}
ModuleData moduleData = new ModuleData()
{
SimpleName = simpleName,
FilePath = filePath,
Module = module,
MappedViewAccessor = mappedViewAccessor
};
lock (this)
{
if (useForBinding)
{
ModuleData actualModuleData = _simpleNameHashtable.AddOrGetExisting(moduleData);
if (actualModuleData != moduleData)
{
if (actualModuleData.FilePath != filePath)
{
throw new FileNotFoundException("Module with same simple name already exists " + filePath);
}
return(actualModuleData.Module);
}
}
mappedViewAccessor = null; // Ownership has been transfered
pdbReader = null; // Ownership has been transferred
_moduleHashtable.AddOrGetExisting(moduleData);
}
return(module);
}
finally
{
if (mappedViewAccessor != null)
{
mappedViewAccessor.Dispose();
}
if (pdbReader != null)
{
pdbReader.Dispose();
}
}
}
public DebugDirectoryEntryNode DebugDirectoryEntry(EcmaModule module, int debugDirEntryIndex)
{
return(_debugDirectoryEntries.GetOrAdd(new ModuleAndIntValueKey(debugDirEntryIndex, module)));
}
private int Run(string[] args)
{
InitializeDefaultOptions();
ArgumentSyntax syntax = ParseCommandLine(args);
if (_help)
{
Help(syntax.GetHelpText());
return(1);
}
if (_outputFilePath == null)
{
throw new CommandLineException("Output filename must be specified (/out <file>)");
}
//
// Set target Architecture and OS
//
if (_targetArchitectureStr != null)
{
if (_targetArchitectureStr.Equals("x86", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X86;
}
else if (_targetArchitectureStr.Equals("x64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.X64;
}
else if (_targetArchitectureStr.Equals("arm", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM;
}
else if (_targetArchitectureStr.Equals("armel", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARMEL;
}
else if (_targetArchitectureStr.Equals("arm64", StringComparison.OrdinalIgnoreCase))
{
_targetArchitecture = TargetArchitecture.ARM64;
}
else
{
throw new CommandLineException("Target architecture is not supported");
}
}
if (_targetOSStr != null)
{
if (_targetOSStr.Equals("windows", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Windows;
}
else if (_targetOSStr.Equals("linux", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.Linux;
}
else if (_targetOSStr.Equals("osx", StringComparison.OrdinalIgnoreCase))
{
_targetOS = TargetOS.OSX;
}
else
{
throw new CommandLineException("Target OS is not supported");
}
}
//
// Initialize type system context
//
SharedGenericsMode genericsMode = _useSharedGenerics || !_isCppCodegen ?
SharedGenericsMode.CanonicalReferenceTypes : SharedGenericsMode.Disabled;
// TODO: compiler switch for SIMD support?
var simdVectorLength = _isCppCodegen ? SimdVectorLength.None : SimdVectorLength.Vector128Bit;
var targetDetails = new TargetDetails(_targetArchitecture, _targetOS, TargetAbi.CoreRT, simdVectorLength);
var typeSystemContext = new CompilerTypeSystemContext(targetDetails, genericsMode);
//
// TODO: To support our pre-compiled test tree, allow input files that aren't managed assemblies since
// some tests contain a mixture of both managed and native binaries.
//
// See: https://github.com/dotnet/corert/issues/2785
//
// When we undo this this hack, replace this foreach with
// typeSystemContext.InputFilePaths = _inputFilePaths;
//
Dictionary <string, string> inputFilePaths = new Dictionary <string, string>();
foreach (var inputFile in _inputFilePaths)
{
try
{
var module = typeSystemContext.GetModuleFromPath(inputFile.Value);
inputFilePaths.Add(inputFile.Key, inputFile.Value);
}
catch (TypeSystemException.BadImageFormatException)
{
// Keep calm and carry on.
}
}
typeSystemContext.InputFilePaths = inputFilePaths;
typeSystemContext.ReferenceFilePaths = _referenceFilePaths;
typeSystemContext.SetSystemModule(typeSystemContext.GetModuleForSimpleName(_systemModuleName));
if (typeSystemContext.InputFilePaths.Count == 0)
{
throw new CommandLineException("No input files specified");
}
//
// Initialize compilation group and compilation roots
//
// Single method mode?
MethodDesc singleMethod = CheckAndParseSingleMethodModeArguments(typeSystemContext);
CompilationModuleGroup compilationGroup;
List <ICompilationRootProvider> compilationRoots = new List <ICompilationRootProvider>();
if (singleMethod != null)
{
// Compiling just a single method
compilationGroup = new SingleMethodCompilationModuleGroup(singleMethod);
compilationRoots.Add(new SingleMethodRootProvider(singleMethod));
}
else
{
// Either single file, or multifile library, or multifile consumption.
EcmaModule entrypointModule = null;
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
if (module.PEReader.PEHeaders.IsExe)
{
if (entrypointModule != null)
{
throw new Exception("Multiple EXE modules");
}
entrypointModule = module;
}
compilationRoots.Add(new ExportedMethodsRootProvider(module));
}
if (entrypointModule != null)
{
LibraryInitializers libraryInitializers =
new LibraryInitializers(typeSystemContext, _isCppCodegen);
compilationRoots.Add(new MainMethodRootProvider(entrypointModule, libraryInitializers.LibraryInitializerMethods));
}
if (_multiFile)
{
List <EcmaModule> inputModules = new List <EcmaModule>();
foreach (var inputFile in typeSystemContext.InputFilePaths)
{
EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value);
if (entrypointModule == null)
{
// This is a multifile production build - we need to root all methods
compilationRoots.Add(new LibraryRootProvider(module));
}
inputModules.Add(module);
}
compilationGroup = new MultiFileSharedCompilationModuleGroup(typeSystemContext, inputModules);
}
else
{
if (entrypointModule == null)
{
throw new Exception("No entrypoint module");
}
compilationRoots.Add(new ExportedMethodsRootProvider((EcmaModule)typeSystemContext.SystemModule));
compilationGroup = new SingleFileCompilationModuleGroup(typeSystemContext);
}
foreach (var rdXmlFilePath in _rdXmlFilePaths)
{
compilationRoots.Add(new RdXmlRootProvider(typeSystemContext, rdXmlFilePath));
}
}
//
// Compile
//
CompilationBuilder builder;
if (_isCppCodegen)
{
builder = new CppCodegenCompilationBuilder(typeSystemContext, compilationGroup);
}
else
{
builder = new RyuJitCompilationBuilder(typeSystemContext, compilationGroup);
}
bool useScanner = _useScanner ||
(_optimizationMode != OptimizationMode.None && !_isCppCodegen);
useScanner &= !_noScanner;
ILScanResults scanResults = null;
if (useScanner)
{
ILScannerBuilder scannerBuilder = builder.GetILScannerBuilder()
.UseCompilationRoots(compilationRoots);
if (_scanDgmlLogFileName != null)
{
scannerBuilder.UseDependencyTracking(_generateFullScanDgmlLog ? DependencyTrackingLevel.All : DependencyTrackingLevel.First);
}
IILScanner scanner = scannerBuilder.ToILScanner();
scanResults = scanner.Scan();
}
var logger = _isVerbose ? new Logger(Console.Out, true) : Logger.Null;
DebugInformationProvider debugInfoProvider = _enableDebugInfo ?
(_ilDump == null ? new DebugInformationProvider() : new ILAssemblyGeneratingMethodDebugInfoProvider(_ilDump, new EcmaOnlyDebugInformationProvider())) :
new NullDebugInformationProvider();
DependencyTrackingLevel trackingLevel = _dgmlLogFileName == null ?
DependencyTrackingLevel.None : (_generateFullDgmlLog ? DependencyTrackingLevel.All : DependencyTrackingLevel.First);
CompilerGeneratedMetadataManager metadataManager = new CompilerGeneratedMetadataManager(compilationGroup, typeSystemContext, _metadataLogFileName);
builder
.UseBackendOptions(_codegenOptions)
.UseMetadataManager(metadataManager)
.UseLogger(logger)
.UseDependencyTracking(trackingLevel)
.UseCompilationRoots(compilationRoots)
.UseOptimizationMode(_optimizationMode)
.UseDebugInfoProvider(debugInfoProvider);
if (scanResults != null)
{
// If we have a scanner, feed the vtable analysis results to the compilation.
// This could be a command line switch if we really wanted to.
builder.UseVTableSliceProvider(scanResults.GetVTableLayoutInfo());
// If we have a scanner, feed the generic dictionary results to the compilation.
// This could be a command line switch if we really wanted to.
builder.UseGenericDictionaryLayoutProvider(scanResults.GetDictionaryLayoutInfo());
}
ICompilation compilation = builder.ToCompilation();
ObjectDumper dumper = _mapFileName != null ? new ObjectDumper(_mapFileName) : null;
CompilationResults compilationResults = compilation.Compile(_outputFilePath, dumper);
if (_dgmlLogFileName != null)
{
compilationResults.WriteDependencyLog(_dgmlLogFileName);
}
if (scanResults != null)
{
SimdHelper simdHelper = new SimdHelper();
if (_scanDgmlLogFileName != null)
{
scanResults.WriteDependencyLog(_scanDgmlLogFileName);
}
// If the scanner and compiler don't agree on what to compile, the outputs of the scanner might not actually be usable.
// We are going to check this two ways:
// 1. The methods and types generated during compilation are a subset of method and types scanned
// 2. The methods and types scanned are a subset of methods and types compiled (this has a chance to hold for unoptimized builds only).
// Check that methods and types generated during compilation are a subset of method and types scanned
bool scanningFail = false;
DiffCompilationResults(ref scanningFail, compilationResults.CompiledMethodBodies, scanResults.CompiledMethodBodies,
"Methods", "compiled", "scanned", method => !(method.GetTypicalMethodDefinition() is EcmaMethod));
DiffCompilationResults(ref scanningFail, compilationResults.ConstructedEETypes, scanResults.ConstructedEETypes,
"EETypes", "compiled", "scanned", type => !(type.GetTypeDefinition() is EcmaType));
// If optimizations are enabled, the results will for sure not match in the other direction due to inlining, etc.
// But there's at least some value in checking the scanner doesn't expand the universe too much in debug.
if (_optimizationMode == OptimizationMode.None)
{
// Check that methods and types scanned are a subset of methods and types compiled
// If we find diffs here, they're not critical, but still might be causing a Size on Disk regression.
bool dummy = false;
// We additionally skip methods in SIMD module because there's just too many intrisics to handle and IL scanner
// doesn't expand them. They would show up as noisy diffs.
DiffCompilationResults(ref dummy, scanResults.CompiledMethodBodies, compilationResults.CompiledMethodBodies,
"Methods", "scanned", "compiled", method => !(method.GetTypicalMethodDefinition() is EcmaMethod) || simdHelper.IsInSimdModule(method.OwningType));
DiffCompilationResults(ref dummy, scanResults.ConstructedEETypes, compilationResults.ConstructedEETypes,
"EETypes", "scanned", "compiled", type => !(type.GetTypeDefinition() is EcmaType));
}
if (scanningFail)
{
throw new Exception("Scanning failure");
}
}
if (debugInfoProvider is IDisposable)
{
((IDisposable)debugInfoProvider).Dispose();
}
return(0);
}
public CopiedMetadataBlobNode CopiedMetadataBlob(EcmaModule module)
{
return(_copiedMetadataBlobs.GetOrAdd(module));
}
public DebugDirectoryEntryNode(EcmaModule module)
{
_module = module;
}
public CopiedStrongNameSignatureNode CopiedStrongNameSignature(EcmaModule module)
{
return(_copiedStrongNameSignatures.GetOrAdd(module));
}
public int GetModuleIndex(EcmaModule module)
{
return(_moduleIndexLookup(module));
}
public CopiedStrongNameSignatureNode(EcmaModule module)
{
_module = module;
}