private static void RegisterKnownWorkspaceResolvers(
StringTable stringTable,
DScriptWorkspaceResolverFactory workspaceFactory,
IFrontEndStatistics statistics)
{
workspaceFactory.RegisterResolver(
KnownResolverKind.SourceResolverKind,
() => new WorkspaceSourceModuleResolver(stringTable, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.DScriptResolverKind,
() => new WorkspaceSourceModuleResolver(stringTable, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.DefaultSourceResolverKind,
() => new WorkspaceDefaultSourceModuleResolver(stringTable, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.NugetResolverKind,
() => new WorkspaceNugetModuleResolver(stringTable, statistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.DownloadResolverKind,
() => new DownloadWorkspaceResolver());
workspaceFactory.RegisterResolver(
KnownResolverKind.MsBuildResolverKind,
() => new MsBuildWorkspaceResolver());
workspaceFactory.RegisterResolver(
KnownResolverKind.NinjaResolverKind,
() => new NinjaWorkspaceResolver());
workspaceFactory.RegisterResolver(
KnownResolverKind.CMakeResolverKind,
() => new CMakeWorkspaceResolver());
}
/// <summary>
/// Parses a given file content.
/// </summary>
public ISourceFile ParseSourceFileContent(
ParsingOptions parsingOptions,
string specPathString,
FileContent content,
Func <TextSource> textSourceProvider,
IFrontEndStatistics frontEndStatistics,
ByteContent serializedAst)
{
frontEndStatistics = frontEndStatistics ?? new FrontEndStatistics();
var parser = serializedAst.IsValid
? new PublicSurfaceParser(Context.PathTable, serializedAst.Content, serializedAst.Length)
: parsingOptions.ConvertPathLikeLiteralsAtParseTime
? new DScriptParser(Context.PathTable)
: new TypeScript.Net.Parsing.Parser();
ISourceFile sourceFile;
using (frontEndStatistics.SpecParsing.Start(specPathString))
{
sourceFile = parser.ParseSourceFileContent(specPathString, CreateTextSource(content), new FuncBasedTextSourceProvider(textSourceProvider), parsingOptions);
}
// add ast-level stats
frontEndStatistics.SourceFileIdentifiers.Increment(sourceFile.IdentifierCount, specPathString);
frontEndStatistics.SourceFileLines.Increment(sourceFile.LineMap.Map.LongLength, specPathString);
frontEndStatistics.SourceFileChars.Increment(content.Length, specPathString);
frontEndStatistics.SourceFileNodes.Increment(sourceFile.NodeCount, specPathString);
return(sourceFile);
}
private static void RegisterKnownWorkspaceResolvers(
DScriptWorkspaceResolverFactory workspaceFactory,
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
MsBuildFrontEnd msBuildFrontEnd,
NinjaFrontEnd ninjaFrontEnd,
CMakeFrontEnd cmakeFrontend)
{
workspaceFactory.RegisterResolver(
KnownResolverKind.SourceResolverKind,
() => new WorkspaceSourceModuleResolver(constants, sharedModuleRegistry, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.DScriptResolverKind,
() => new WorkspaceSourceModuleResolver(constants, sharedModuleRegistry, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.DefaultSourceResolverKind,
() => new WorkspaceDefaultSourceModuleResolver(constants, sharedModuleRegistry, statistics, logger: null));
workspaceFactory.RegisterResolver(
KnownResolverKind.NugetResolverKind,
() => new WorkspaceNugetModuleResolver(constants, sharedModuleRegistry, statistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.DownloadResolverKind,
() => new DownloadWorkspaceResolver(constants, sharedModuleRegistry));
workspaceFactory.RegisterResolver(
KnownResolverKind.MsBuildResolverKind,
() => new MsBuildWorkspaceResolver(constants, sharedModuleRegistry, statistics, msBuildFrontEnd));
workspaceFactory.RegisterResolver(
KnownResolverKind.NinjaResolverKind,
() => new NinjaWorkspaceResolver(constants, sharedModuleRegistry, statistics, ninjaFrontEnd));
workspaceFactory.RegisterResolver(
KnownResolverKind.CMakeResolverKind,
() => new CMakeWorkspaceResolver(constants, sharedModuleRegistry, statistics, cmakeFrontend, ninjaFrontEnd));
}
/// <nodoc/>
public FrontEndPublicFacadeAndAstProvider(
FrontEndEngineAbstraction engine,
Logger logger,
LoggingContext loggingContext,
string frontEndEngineDirectory,
bool logFrontEndStatistics,
PathTable pathTable,
IFrontEndStatistics statistics,
CancellationToken cancellationToken)
{
Contract.Requires(engine != null);
Contract.Requires(loggingContext != null);
Contract.Requires(!string.IsNullOrEmpty(frontEndEngineDirectory));
Contract.Requires(pathTable != null);
Contract.Requires(statistics != null);
m_engine = engine;
m_pathTable = pathTable;
m_statistics = statistics;
m_fileCombiner = new FileCombiner(
loggingContext,
SpecCacheFullPath(frontEndEngineDirectory),
FileCombiner.FileCombinerUsage.IncrementalScriptFrontEnd,
logFrontEndStatistics);
m_logger = logger;
m_loggingContext = loggingContext;
var queueOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, CancellationToken = cancellationToken
};
Action <FileContentWithHash> action = SaveFile;
m_filesToSaveQueue = new ActionBlock <FileContentWithHash>(action, queueOptions);
}
private static IFrontEndController TryCreateFrontEndController(
FrontEndFactory frontEndFactory,
IDecorator <EvaluationResult> decorator,
ICommandLineConfiguration configuration,
SymbolTable symbolTable,
LoggingContext loggingContext,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);
// Statistic should be global for all front-ends, not per an instance.
var frontEndStatistics = statistics ?? new FrontEndStatistics();
var sharedModuleRegistry = new ModuleRegistry(symbolTable);
// Note, that the following code is absolutely critical for detecting that front-end related objects
// are freed successfully after evaluation.
// ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);
frontEndFactory.SetConfigurationProcessor(
new ConfigurationProcessor(
new FrontEndStatistics(), // Configuration processing is so lightweight that it won't affect overall perf statistics
logger: null));
frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new NugetFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new DownloadFrontEnd());
#if PLATFORM_WIN
frontEndFactory.AddFrontEnd(new MsBuildFrontEnd());
frontEndFactory.AddFrontEnd(new NinjaFrontEnd());
frontEndFactory.AddFrontEnd(new CMakeFrontEnd());
frontEndFactory.AddFrontEnd(new RushFrontEnd());
frontEndFactory.AddFrontEnd(new YarnFrontEnd());
frontEndFactory.AddFrontEnd(new LageFrontEnd());
#endif
if (!frontEndFactory.TrySeal(loggingContext))
{
return(null);
}
return(new FrontEndHostController(
frontEndFactory,
evaluationScheduler: EvaluationScheduler.Default,
moduleRegistry: sharedModuleRegistry,
frontEndStatistics: frontEndStatistics,
logger: BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
collector: collector,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
}
/// <nodoc />
public FrontEndCache(string outputFolder, Logger logger, LoggingContext loggingContext, IFrontEndStatistics frontEndStatistics, PathTable pathTable)
{
m_logger = logger;
m_loggingContext = loggingContext;
m_frontEndStatistics = frontEndStatistics;
m_pathTable = pathTable;
m_cacheFileName = Path.Combine(outputFolder, CacheFileName);
}
/// <nodoc/>
public WorkspaceDefaultSourceModuleResolver(
StringTable stringTable,
IFrontEndStatistics statistics,
Logger logger = null)
: base(stringTable, statistics, logger)
{
Name = nameof(WorkspaceDefaultSourceModuleResolver);
}
protected DScriptInterpreterBase(IFrontEndStatistics statistics, Logger logger,
FrontEndHost host, FrontEndContext context, IConfiguration configuration)
: this(statistics, logger)
{
// ReSharper disable once VirtualMemberCallInConstructor
// The call is safe and used only for testing purposes.
InitializeInterpreter(host, context, configuration);
}
/// <nodoc/>
public DScriptFrontEnd(
IFrontEndStatistics statistics,
Logger logger = null,
IDecorator <EvaluationResult> evaluationDecorator = null)
: base(statistics, logger)
{
m_customLogger = logger;
m_evaluationDecorator = evaluationDecorator;
}
/// <nodoc/>
public WorkspaceDefaultSourceModuleResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
Logger logger = null)
: base(constants, sharedModuleRegistry, statistics, logger)
{
Name = nameof(WorkspaceDefaultSourceModuleResolver);
}
/// <nodoc />
protected DScriptInterpreterBase(IFrontEndStatistics statistics, Logger logger)
{
Contract.Requires(statistics != null);
Name = "DScript";
FrontEndStatistics = statistics;
Logger = logger ?? Logger.CreateLogger();
Statistics = new EvaluationStatistics();
}
/// <nodoc/>
public MsBuildFrontEnd(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
Logger logger = null)
: base(constants, sharedModuleRegistry, statistics, logger)
{
Name = nameof(MsBuildFrontEnd);
}
/// <inheritdoc/>
public CMakeWorkspaceResolver(
StringTable stringTable,
IFrontEndStatistics statistics)
: base(statistics, logger: null)
{
Name = nameof(CMakeWorkspaceResolver);
m_relativePathToCMakeRunner = RelativePath.Create(stringTable, CMakeRunnerRelativePath);
EmbeddedNinjaWorkspaceResolver = new NinjaWorkspaceResolver(stringTable, statistics);
m_embeddedResolverSettings = new Lazy <NinjaResolverSettings>(CreateEmbeddedResolverSettings);
}
/// <inheritdoc/>
public NinjaWorkspaceResolver(
StringTable stringTable,
IFrontEndStatistics statistics)
: base(statistics, logger: null)
{
Name = nameof(NinjaWorkspaceResolver);
m_relativePathToGraphConstructionTool = RelativePath.Create(stringTable, NinjaGraphBuilderRelativePath);
m_graph = new Lazy <Task <Possible <NinjaGraphWithModuleDefinition> > >(TryComputeBuildGraphAsync);
SerializedGraphPath = new Lazy <AbsolutePath>(GetToolOutputPath);
}
/// <nodoc/>
public DScriptFrontEnd(
IFrontEndStatistics statistics,
Logger logger = null,
IDecorator <EvaluationResult> evaluationDecorator = null)
: base(statistics, logger)
{
Name = nameof(DScriptFrontEnd);
m_evaluationDecorator = evaluationDecorator;
}
/// <inheritdoc/>
public MsBuildWorkspaceResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
MsBuildFrontEnd frontEnd)
: base(constants, sharedModuleRegistry, statistics, logger: null)
{
Name = nameof(MsBuildWorkspaceResolver);
m_frontEnd = frontEnd;
}
/// <nodoc />
public NugetResolver(
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics,
SourceFileProcessingQueue <bool> parseQueue,
Logger logger = null,
IDecorator <EvaluationResult> evaluationDecorator = null)
: base(host, context, configuration, statistics, parseQueue, logger, evaluationDecorator)
{
}
/// <nodoc/>
public DScriptFrontEnd(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
Logger logger = null,
IDecorator <EvaluationResult> evaluationDecorator = null)
: base(constants, sharedModuleRegistry, statistics, logger)
{
Name = nameof(DScriptFrontEnd);
m_evaluationDecorator = evaluationDecorator;
}
/// <inheritdoc/>
public NinjaWorkspaceResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
NinjaFrontEnd frontEnd)
: base(constants, sharedModuleRegistry, statistics, logger: null)
{
Name = nameof(NinjaWorkspaceResolver);
m_frontEnd = frontEnd;
m_relativePathToGraphConstructionTool = RelativePath.Create(frontEnd.Context.StringTable, NinjaGraphBuilderRelativePath);
m_graph = new Lazy <Task <Possible <NinjaGraphWithModuleDefinition> > >(TryComputeBuildGraphAsync);
SerializedGraphPath = new Lazy <AbsolutePath>(GetToolOutputPath);
}
/// <nodoc />
public NugetResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics,
SourceFileProcessingQueue <bool> parseQueue,
Logger logger = null,
IDecorator <EvaluationResult> evaluationDecorator = null)
: base(constants, sharedModuleRegistry, host, context, configuration, statistics, parseQueue, logger, evaluationDecorator)
{
}
/// <nodoc />
protected DScriptInterpreterBase(GlobalConstants constants, ModuleRegistry sharedModuleRegistry, IFrontEndStatistics statistics, Logger logger)
{
Contract.Requires(constants != null);
Contract.Requires(sharedModuleRegistry != null);
Contract.Requires(statistics != null);
Constants = constants;
SharedModuleRegistry = sharedModuleRegistry;
Name = "DScript";
FrontEndStatistics = statistics;
Logger = logger ?? Logger.CreateLogger();
Statistics = new EvaluationStatistics();
}
/// <nodoc/>
public NinjaResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
Logger logger,
string frontEndName)
: base(constants, sharedModuleRegistry, statistics, logger, host, context, configuration)
{
Contract.Requires(!string.IsNullOrEmpty(frontEndName));
m_frontEndName = frontEndName;
}
/// <inheritdoc/>
public CMakeWorkspaceResolver(
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
IFrontEndStatistics statistics,
CMakeFrontEnd frontEnd,
NinjaFrontEnd ninjaFrontEnd)
: base(constants, sharedModuleRegistry, statistics, logger: null)
{
Name = nameof(CMakeWorkspaceResolver);
m_frontEnd = frontEnd;
m_relativePathToCMakeRunner = RelativePath.Create(frontEnd.Context.StringTable, CMakeRunnerRelativePath);
EmbeddedNinjaWorkspaceResolver = new NinjaWorkspaceResolver(constants, sharedModuleRegistry, statistics, ninjaFrontEnd);
m_embeddedResolverSettings = new Lazy <NinjaResolverSettings>(CreateEmbeddedResolverSettings);
}
/// <nodoc/>
public RuntimeModelFactory(
IFrontEndStatistics statistics,
AstConversionConfiguration conversionConfiguration,
DiagnosticAnalyzer linter,
Workspace workspace)
{
Contract.Requires(statistics != null);
Contract.Requires(conversionConfiguration != null);
m_statistics = statistics;
m_conversionConfiguration = conversionConfiguration;
m_linter = linter;
m_workspace = workspace;
}
/// <nodoc />
public ConfigurationConversionHelper(
[CanBeNull] FrontEndEngineAbstraction engine,
ConfigurationKind kind,
Logger logger,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics)
: base(statistics, logger, host, context, configuration)
{
Engine = engine ?? new SimpleFrontEndEngineAbstraction(context.PathTable, context.FileSystem, configuration);
Kind = kind;
ConversionConfiguration = AstConversionConfiguration.ForConfiguration(FrontEndConfiguration);
Linter = Lazy.Create(() => CreateLinter(ConversionConfiguration));
}
/// <nodoc />
public DScriptSourceResolver(
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics,
SourceFileProcessingQueue <bool> parseQueue,
Logger logger = null,
IDecorator <Values.EvaluationResult> evaluationDecorator = null)
: base(statistics, logger, host, context, configuration)
{
Contract.Requires(parseQueue != null);
m_parseQueue = parseQueue;
m_evaluationDecorator = evaluationDecorator;
}
/// <nodoc />
private FrontEndControllerFactory(
FrontEndMode mode,
LoggingContext loggingContext,
ICommandLineConfiguration configuration,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
m_mode = mode;
CollectMemoryAsSoonAsPossible = collectMemoryAsSoonAsPossible;
Configuration = configuration;
LoggingContext = loggingContext;
Collector = collector;
m_statistics = statistics;
}
/// <nodoc />
public static FrontEndControllerFactory Create(
FrontEndMode mode,
LoggingContext loggingContext,
ICommandLineConfiguration configuration,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible = true,
IFrontEndStatistics statistics = null)
{
return(new FrontEndControllerFactory(
mode,
loggingContext,
configuration,
collector,
collectMemoryAsSoonAsPossible,
statistics));
}
/// <nodoc/>
public RuntimeModelFactory(
Logger logger,
LoggingContext loggingContext,
IFrontEndStatistics statistics,
AstConversionConfiguration conversionConfiguration,
[CanBeNull] Workspace workspace)
: this(
statistics,
conversionConfiguration,
DiagnosticAnalyzer.Create(
logger,
loggingContext,
new HashSet <string>(conversionConfiguration.PolicyRules),
conversionConfiguration.DisableLanguagePolicies),
workspace)
{
}
/// <nodoc />
public ConfigurationConversionHelper(
[CanBeNull] FrontEndEngineAbstraction engine,
ConfigurationKind kind,
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
Logger logger,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics = null)
: base(constants, sharedModuleRegistry, statistics ?? new FrontEndStatistics(), logger, host, context, configuration)
{
Engine = engine ?? new SimpleFrontEndEngineAbstraction(context.PathTable, context.FileSystem, configuration);
Kind = kind;
ConversionConfiguration = AstConversionConfiguration.ForConfiguration(FrontEndConfiguration);
Linter = Lazy.Create(() => CreateLinter(ConversionConfiguration));
}
