// Performs core runtime actions
internal static IProgram Build(ICommandLineConfiguration configuration,
IEnumerable <string> arguments,
out object options)
{
Check.NotNull(arguments, nameof(arguments));
Check.NotNull(configuration, nameof(configuration));
// Create parse context
var parseContext = new ParseContext(arguments);
// Aggregate commands
var aggregatedRuntime = new AggregatedRuntimeCommand(parseContext,
configuration.GetRuntimeCommand(),
configuration.SubConfigurations);
// Create options instance
options = aggregatedRuntime.GetOptions();
// Map arguments
var result = aggregatedRuntime.MapArguments(parseContext, options, default(ParserType));
// Show help?
if ((result & ContextResult.Help) == ContextResult.Help)
{
return(new HelpProgram(
// Will throw ConfigurationException
configuration.HelpWriter !,
aggregatedRuntime.HelpContentProvider !));
}
CheckContextStragglers(parseContext);
return(aggregatedRuntime.SelectedRuntime);
}
private static CommandLineConfiguration GetCommandLineConfiguration(
ICommandLineConfiguration commandLineConfig,
WorkspaceBuilderConfiguration configuration)
{
return(new CommandLineConfiguration(commandLineConfig)
{
FrontEnd =
{
DebugScript = false,
PreserveFullNames = true,
PreserveTrivia = true, // We want to preserve comments, so let's not skip trivia
CancelParsingOnFirstFailure = configuration.CancelOnFirstParsingFailure,
UseSpecPublicFacadeAndAstWhenAvailable = configuration.PublicFacadeOptimization,
ConstructAndSaveBindingFingerprint = configuration.SaveBindingFingerprint,
NameResolutionSemantics = NameResolutionSemantics.ImplicitProjectReferences,
// If SkipNuget is specified, then all the packages should be on disk.
// Skipping nuget restore in this case.
UsePackagesFromFileSystem = configuration.SkipNuget,
// Don't release workspace so that analyses can still be done if the min required phase is evaluation.
// TODO: Hack -- when phase Evaluate is use, then release workspace. This is for Office to be performant.
ReleaseWorkspaceBeforeEvaluation = !commandLineConfig.Engine.Phase.HasFlag(EnginePhases.Evaluate),
AllowUnsafeAmbient = true,
},
Logging =
{
LogsToRetain = 0,
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled
}
});
}
private bool YarnInit(ICommandLineConfiguration config)
{
// Create a package.json, root of all the workspaces. This package needs to be private
// since workspaces need to be declared in a private one
var result = YarnRun(config, "init --private --yes");
if (!result)
{
return(false);
}
// Update the root package.json to enable workspaces
var pathToPackageJson = config.Layout.SourceDirectory.Combine(PathTable, "package.json").ToString(PathTable);
string mainJson = File.ReadAllText(pathToPackageJson);
int closingBracket = mainJson.LastIndexOf('}');
mainJson = mainJson.Insert(closingBracket, @",
""workspaces"": {
""packages"": [
""src/*""
]}");
File.WriteAllText(pathToPackageJson, mainJson);
return(YarnRun(config, "install"));
}
private bool YarnRun(ICommandLineConfiguration config, string yarnArgs)
{
string arguments = $"{PathToYarn}.js {yarnArgs}";
string filename = PathToNode;
// Unfortunately, capturing standard out/err non-deterministically hangs node.exe on exit when
// concurrent npm install operations happen. Found reported bugs about this that look similar enough
// to the problem that manifested here.
// So we just report exit codes.
var startInfo = new ProcessStartInfo
{
FileName = filename,
Arguments = arguments,
WorkingDirectory = config.Layout.SourceDirectory.ToString(PathTable),
RedirectStandardError = false,
RedirectStandardOutput = false,
UseShellExecute = false,
};
startInfo.Environment["PATH"] += $";{PathToNodeFolder}";
startInfo.Environment["APPDATA"] = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
var runYarn = Process.Start(startInfo);
runYarn.WaitForExit();
return(runYarn.ExitCode == 0);
}
protected virtual BuildXLEngineResult CreateAndRunEngine(
ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
out BuildXLEngine engine,
Action <EngineTestHooksData> verifyEngineTestHooksData = null,
TestCache testCache = null)
{
testCache = testCache ?? TestCache;
using (EngineTestHooksData testHooks = new EngineTestHooksData
{
AppDeployment = appDeployment,
CacheFactory = (context) => new EngineCache(
testCache.GetArtifacts(context),
testCache.Fingerprints)
})
{
engine = CreateEngine(config, appDeployment, testRootDirectory, rememberAllChangedTrackedInputs, verifyEngineTestHooksData);
// Ignore DX222 for csc.exe being outside of src directory
IgnoreWarnings();
engine.TestHooks = testHooks;
var result = engine.RunForFrontEndTests(LoggingContext);
return(result);
}
}
protected BuildXLEngineResult RunYarnProjects(
ICommandLineConfiguration config,
TestCache testCache = null,
IDetoursEventListener detoursListener = null)
{
// This bootstraps the 'repo'
if (!YarnInit(config))
{
throw new InvalidOperationException("Yarn init failed.");
}
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
((CommandLineConfiguration)config).Engine.Phase = Phase;
((CommandLineConfiguration)config).Sandbox.FileSystemMode = FileSystemMode.RealAndMinimalPipGraph;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out var engine,
testCache: testCache,
detoursListener: detoursListener);
return(engineResult);
}
}
protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);
var factory = FrontEndControllerFactory.Create(
FrontEndMode.NormalMode,
LoggingContext,
config,
// Set the timeout to a large number to avoid useless performance collections in tests.
new PerformanceCollector(TimeSpan.FromHours(1)),
collectMemoryAsSoonAsPossible: false);
var engine = BuildXLEngine.Create(
LoggingContext,
engineContext,
config,
factory,
rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
);
return(engine);
}
protected FrontEndHostController RunEngineAndGetFrontEndHostController(
ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var result = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory,
rememberAllChangedTrackedInputs,
out var engine,
verifyEngineTestHooksData);
m_frontEndContext = engine.Context.ToFrontEndContext(LoggingContext);
verifyEngineTestHooksData?.Invoke(engine.TestHooks);
// If the engine reloaded and created a new pipGraph we need to udpate the Test file system with that new pipgraph as well
if (!engine.TestHooks.GraphReuseResult.IsNoReuse)
{
FileSystem = (IMutableFileSystem)engine.Context.FileSystem;
}
if (!result.IsSuccess)
{
Assert.True(false, $"Failed to run the engine. See '{testRootDirectory ?? TestOutputDirectory}' for more details.");
}
return((FrontEndHostController)engine.FrontEndController);
}
private bool CreateFactories(
FrontEndContext frontEndContext,
TestEngineAbstraction engineAbstraction,
FrontEndStatistics frontEndStatistics,
ICommandLineConfiguration configuration,
out AmbientTesting ambientTesting,
out ModuleRegistry moduleRegistry,
out FrontEndFactory frontEndFactory)
{
moduleRegistry = new ModuleRegistry(frontEndContext.SymbolTable);
ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, moduleRegistry.PrimitiveTypes);
ambientTesting.Initialize(moduleRegistry.GlobalLiteral);
var ambientAssert = new AmbientAssert(moduleRegistry.PrimitiveTypes);
ambientAssert.Initialize(moduleRegistry.GlobalLiteral);
// Create the controller
frontEndFactory = new FrontEndFactory();
frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
frontEndFactory.AddFrontEnd(
new DScriptFrontEnd(
frontEndStatistics,
logger: m_astLogger));
if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
{
HandleDiagnostics();
return(false);
}
return(true);
}
/// <summary>
/// Executes the application asynchronously.
/// </summary>
/// <param name="configuration">Configuration instance.</param>
/// <param name="arguments">Arguments given on the command line.</param>
public static Task RunAsync(ICommandLineConfiguration configuration,
IEnumerable <string> arguments)
{
return(RuntimeCommandBuilder
.Build(configuration, arguments, out var options)
.InvokeAsync(options, CancellationToken.None));
}
/// <summary>
/// Executes the application.
/// </summary>
/// <param name="configuration">Configuration instance.</param>
/// <param name="arguments">Arguments given on the command line.</param>
public static void Run(ICommandLineConfiguration configuration,
IEnumerable <string> arguments)
{
RuntimeCommandBuilder
.Build(configuration, arguments, out var options)
.Invoke(options);
}
private static IFrontEndController TryCreateFrontEndController(
FrontEndFactory frontEndFactory,
IDecorator <EvaluationResult> decorator,
ICommandLineConfiguration configuration,
SymbolTable symbolTable,
LoggingContext loggingContext,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);
// Statistic should be global for all front-ends, not per an instance.
var frontEndStatistics = statistics ?? new FrontEndStatistics();
var sharedModuleRegistry = new ModuleRegistry(symbolTable);
// Note, that the following code is absolutely critical for detecting that front-end related objects
// are freed successfully after evaluation.
// ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);
frontEndFactory.SetConfigurationProcessor(
new ConfigurationProcessor(
new FrontEndStatistics(), // Configuration processing is so lightweight that it won't affect overall perf statistics
logger: null));
frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new NugetFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new DownloadFrontEnd());
#if PLATFORM_WIN
frontEndFactory.AddFrontEnd(new MsBuildFrontEnd());
frontEndFactory.AddFrontEnd(new NinjaFrontEnd());
frontEndFactory.AddFrontEnd(new CMakeFrontEnd());
frontEndFactory.AddFrontEnd(new RushFrontEnd());
frontEndFactory.AddFrontEnd(new YarnFrontEnd());
frontEndFactory.AddFrontEnd(new LageFrontEnd());
#endif
if (!frontEndFactory.TrySeal(loggingContext))
{
return(null);
}
return(new FrontEndHostController(
frontEndFactory,
evaluationScheduler: EvaluationScheduler.Default,
moduleRegistry: sharedModuleRegistry,
frontEndStatistics: frontEndStatistics,
logger: BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
collector: collector,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
}
private Diagnostic ParseConfigurationWithFailure(string configuration, ICommandLineConfiguration commandLine = null)
{
var result = ParseConfiguration(configuration, commandLine);
Assert.True(result.HasError);
return(result.Errors.First());
}
/// <summary>
/// Parses the arguments without invoking the client handler.
/// </summary>
/// <typeparam name="TOptions">Options type.</typeparam>
/// <param name="configuration">Configuration</param>
/// <param name="arguments">Arguments given on the command line</param>
/// <returns>Populated options object</returns>
public static TOptions ParseArguments <TOptions>(ICommandLineConfiguration configuration,
IEnumerable <string> arguments)
where TOptions : class
{
RuntimeCommandBuilder.Build(configuration, arguments, out var options);
return((TOptions)options);
}
private IConfiguration ParseConfigurationSuccessfully(string configuration, ICommandLineConfiguration commandLine = null)
{
var result = ParseConfiguration(configuration, commandLine);
Assert.False(result.HasError);
return(result.Result);
}
private void RunAndAssertGraphCacheHit(ICommandLineConfiguration config, AppDeployment appDeployment, bool rememberAllChangedTrackedInputs = false)
{
using (var hostController = RunEngineAndGetFrontEndHostController(config, appDeployment, null, rememberAllChangedTrackedInputs))
{
AssertNotLogged(LogEventId.EndSerializingPipGraph);
XAssert.IsNull(hostController.Workspace);
}
}
private Process RunEngineAndRetrieveProcess(ICommandLineConfiguration config)
{
var engineResult = RunEngineWithConfig(config);
Assert.True(engineResult.IsSuccess);
var pipGraph = engineResult.EngineState.PipGraph;
var process = (Process)pipGraph.RetrievePipsOfType(PipType.Process).Single();
return(process);
}
private FrontEndPublicFacadeAndAstProvider GetPublicFacadeProvider(ICommandLineConfiguration config)
{
return(new FrontEndPublicFacadeAndAstProvider(
new BasicFrontEndEngineAbstraction(PathTable, FileSystem),
LoggingContext,
config.Layout.EngineCacheDirectory.ToString(PathTable),
config.FrontEnd.LogStatistics,
PathTable,
new FrontEndStatistics(),
new CancellationToken(false)));
}
protected IList<AbsolutePath> RunAndRetrieveSpecs(ICommandLineConfiguration config, AppDeployment appDeployment, bool rememberAllChangedTrackedInputs = false)
{
IList<AbsolutePath> specs;
using (var controller = RunEngineAndGetFrontEndHostController(config, appDeployment, null, rememberAllChangedTrackedInputs))
{
var workspace = controller.Workspace;
specs = workspace.GetAllSpecFiles().Where(spec => !workspace.PreludeModule.Specs.ContainsKey(spec)).ToList();
}
Assert.True(specs != null);
return specs;
}
protected BuildXLEngineResult RunYarnProjects(
ICommandLineConfiguration config,
TestCache testCache = null,
IDetoursEventListener detoursListener = null)
{
// This bootstraps the 'repo'
if (!YarnInit(config.Layout.SourceDirectory))
{
throw new InvalidOperationException("Yarn init failed.");
}
return(RunEngine(config, testCache, detoursListener));
}
private void AssertCongruent(PathTable pathTable, ICommandLineConfiguration commandLineConfig, LightConfig lightConfig)
{
XAssert.AreEqual(lightConfig.AnimateTaskbar, commandLineConfig.Logging.AnimateTaskbar);
XAssert.AreEqual(lightConfig.Color, commandLineConfig.Logging.Color);
AssertPathCongruent(pathTable, lightConfig.Config, commandLineConfig.Startup.ConfigFile);
XAssert.AreEqual(lightConfig.FancyConsole, commandLineConfig.Logging.FancyConsole);
XAssert.AreEqual(lightConfig.Help, commandLineConfig.Help);
XAssert.AreEqual(lightConfig.NoLogo, commandLineConfig.NoLogo);
XAssert.AreEqual(lightConfig.Server, commandLineConfig.Server);
XAssert.AreEqual(lightConfig.DisablePathTranslation, commandLineConfig.Logging.DisableLoggedPathTranslation);
AssertPathCongruent(pathTable, lightConfig.ServerDeploymentDirectory, commandLineConfig.ServerDeploymentDirectory);
AssertPathCongruent(pathTable, lightConfig.SubstSource, commandLineConfig.Logging.SubstSource);
AssertPathCongruent(pathTable, lightConfig.SubstTarget, commandLineConfig.Logging.SubstTarget);
}
private bool CreateFactories(
FrontEndContext frontEndContext,
TestEngineAbstraction engineAbstraction,
FrontEndStatistics frontEndStatistics,
ICommandLineConfiguration configuration,
out AmbientTesting ambientTesting,
out DScriptWorkspaceResolverFactory workspaceFactory,
out FrontEndFactory frontEndFactory)
{
var globalConstants = new GlobalConstants(frontEndContext.SymbolTable);
ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, globalConstants.KnownTypes);
ambientTesting.Initialize(globalConstants.Global);
var ambientAssert = new AmbientAssert(globalConstants.KnownTypes);
ambientAssert.Initialize(globalConstants.Global);
var sharedModuleRegistry = new ModuleRegistry();
workspaceFactory = new DScriptWorkspaceResolverFactory();
workspaceFactory.RegisterResolver(
KnownResolverKind.DScriptResolverKind,
() => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.SourceResolverKind,
() => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.DefaultSourceResolverKind,
() => new WorkspaceDefaultSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
// Create the controller
frontEndFactory = new FrontEndFactory();
frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
frontEndFactory.AddFrontEnd(
new DScriptFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics,
logger: m_astLogger));
if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
{
HandleDiagnostics();
return(false);
}
return(true);
}
/// <nodoc />
private FrontEndControllerFactory(
FrontEndMode mode,
LoggingContext loggingContext,
ICommandLineConfiguration configuration,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
m_mode = mode;
CollectMemoryAsSoonAsPossible = collectMemoryAsSoonAsPossible;
Configuration = configuration;
LoggingContext = loggingContext;
Collector = collector;
m_statistics = statistics;
}
/// <summary>
/// Displays help, bypassing any invocation of client program handlers.
/// </summary>
/// <param name="configuration">Configuration instance</param>
/// <param name="command">The help context - if showing help for the root application,
/// leave as null.</param>
public static void ShowHelp(ICommandLineConfiguration configuration,
string?command = null)
{
if (configuration.HelpTemplate == null)
{
throw ConfigurationExceptions.NoHelpOptionDefined();
}
var helpToken = configuration.HelpTemplate.Tokens.First().DistinguishedForm !;
var args = string.IsNullOrWhiteSpace(command)
? new[] { helpToken }
: new[] { command, helpToken };
Run(configuration, args !);
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
return(TryBuildWorkspaceInternal(commandLineConfig, frontEndContext, engineContext, evaluationFilter, progressHandler, out workspace, out frontEndHostController, out pipGraph,
configuration, forIDE: false, frontEndEngineAbstraction, collectMemoryAsSoonAsPossible));
}
/// <summary>
/// Create new mutable instance from template
/// </summary>
/// <remarks>
/// This is the only class with CommandLineConfiguration as the configuration entrypoint where the pathRemapper is an argument with default value.
/// If the argument was optional everywhere as well there would be no compiler helping us if someone forgot to pass it along.
/// This is the main entrypoint so we allow a default value here for convenience
/// </remarks>
public CommandLineConfiguration(ICommandLineConfiguration template, PathRemapper pathRemapper = null)
: base(template, pathRemapper)
{
Contract.Assume(template != null);
pathRemapper = pathRemapper ?? new PathRemapper();
Help = template.Help;
NoLogo = template.NoLogo;
LaunchDebugger = template.LaunchDebugger;
Startup = new StartupConfiguration(template.Startup, pathRemapper);
Filter = template.Filter;
Server = template.Server;
ServerDeploymentDirectory = pathRemapper.Remap(template.ServerDeploymentDirectory);
ServerMaxIdleTimeInMinutes = template.ServerMaxIdleTimeInMinutes;
}
/// <nodoc />
public static FrontEndControllerFactory Create(
FrontEndMode mode,
LoggingContext loggingContext,
ICommandLineConfiguration configuration,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible = true,
IFrontEndStatistics statistics = null)
{
return(new FrontEndControllerFactory(
mode,
loggingContext,
configuration,
collector,
collectMemoryAsSoonAsPossible,
statistics));
}
public void TestBooleanProperty(string configurationLiteral, bool?originalValue, bool?expectedValue)
{
string code = string.Format(CultureInfo.InvariantCulture, @"
config({{
{0}
}});", configurationLiteral);
ICommandLineConfiguration commandLine = originalValue != null
? new CommandLineConfiguration(new CommandLineConfiguration()
{
InCloudBuild = originalValue.Value
})
: null;
IConfiguration configuration = ParseConfigurationSuccessfully(code, commandLine);
Assert.Equal(expectedValue, configuration.InCloudBuild);
}
/// <inheritdoc />
IConfiguration IConfigurationProcessor.InterpretConfiguration(
AbsolutePath primaryConfigurationFile,
ICommandLineConfiguration commandLineConfiguration)
{
Contract.Requires(primaryConfigurationFile.IsValid);
Contract.Requires(commandLineConfiguration != null);
var configObjectLiteral = ParseAndInterpretConfigFile(primaryConfigurationFile);
if (configObjectLiteral == null)
{
// Error has been reported already
return(null);
}
// Apply Additional configurations from the commandline
foreach (var additionalConfigurationFile in commandLineConfiguration.Startup.AdditionalConfigFiles)
{
configObjectLiteral = ParseAndInterpretConfigFile(additionalConfigurationFile, configObjectLiteral);
}
// TODO: user override is not really working now. Fix me!
try
{
// Merge the object literal with the initial C# defaults.
return(ConfigurationConverter.AugmentConfigurationWith(Context, commandLineConfiguration, configObjectLiteral));
}
catch (ConversionException conversionException)
{
var configFileString = primaryConfigurationFile.ToString(Context.PathTable);
Logger.ReportConversionException(
Context.LoggingContext,
new Location()
{
File = configFileString
},
Name,
GetConversionExceptionMessage(primaryConfigurationFile, conversionException));
return(null);
}
}
protected BuildXLEngineResult RunEngineWithConfig(ICommandLineConfiguration config, TestCache testCache = null)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// Set the specified phase
((CommandLineConfiguration)config).Engine.Phase = Phase;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out var engine,
testCache: testCache);
return(engineResult);
}
}