public XmlReadingContext(AbsolutePath path, PathTable pathTable, StringTable stringTable)
: base(stringTable)
{
Path = path;
PathTable = pathTable;
}
/// <summary>
/// Create a simple wrapper representing the real file system
/// </summary>
public EngineFileSystem(PathTable pathTable, FrontEndEngineAbstraction engine)
: base(pathTable)
{
Contract.Requires(pathTable != null);
m_engine = engine;
}
public PageablePipStore(PathTable pathTable, SymbolTable symbolTable, int initialBufferSize, bool debug)
: base(pathTable, symbolTable, initialBufferSize, debug)
{
}
/// <inheritdoc/>
public string ToDisplayString(PathTable table, AbsolutePath currentFolder)
{
return(Value.ToString(table.StringTable));
}
/// <summary>
/// Creates an instance of <see cref="FailureRecoveryAggregator"/> specific to BuildXL.
/// </summary>
/// <param name="loggingContext">Logging context.</param>
/// <param name="pathTable">Path table.</param>
/// <param name="configuration">Configuration.</param>
/// <returns>An instance of <see cref="FailureRecoveryAggregator"/>.</returns>
public static FailureRecoveryAggregator Create(LoggingContext loggingContext, PathTable pathTable, IConfiguration configuration)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(pathTable.IsValid);
Contract.Requires(configuration != null);
return(FailureRecoveryAggregator.Create(
loggingContext,
new FailureRecovery[]
{
new CorruptedMemosDbRecovery(pathTable, configuration),
new CatastrophicFailureRecovery(pathTable, configuration, loggingContext)
}));
}
/// <summary>
/// Builds a workspace and uses filter to find specs to evaluate.
/// </summary>
public static bool TryCollectFilesToAnalyze(
Tracing.Logger logger,
PathTable pathTable,
EnginePhases phase,
string configFile,
string filter,
out Workspace workspace,
out IReadOnlyDictionary <AbsolutePath, ISourceFile> filesToAnalyze,
out FrontEndContext context)
{
workspace = null;
filesToAnalyze = null;
var loggingContext = new LoggingContext("DScriptAnalyzer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string into EvaluationFilter
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrEmpty(filter))
{
if (!TryGetEvaluationFilter(logger, loggingContext, engineContext, filter, out evaluationFilter))
{
// Error has been reported already
return(false);
}
}
var configFilePath = AbsolutePath.Create(pathTable, configFile);
// Try parsing the workspace from config and evaluation filter
if (!TryBuildWorkspace(
phase,
context,
engineContext,
configFilePath,
evaluationFilter,
progressHandler: null,
workspace: out workspace,
frontEndHostController: out _,
configuration: GetDefaultConfiguration()))
{
return(false);
}
// Find strict subset of specs in workspace that should be analyzed
var collectedFilesToAnalyze = CollectFilesToAnalyze(
workspace,
pathTable,
configFilePath,
evaluationFilter);
if (collectedFilesToAnalyze.Count == 0)
{
logger.ErrorFilterHasNoMatchingSpecs(loggingContext, filter);
return(false);
}
filesToAnalyze = collectedFilesToAnalyze;
return(true);
}
private static AbsolutePath TryFindConfig(AbsolutePath startDir, FrontEndEngineAbstraction engine, PathTable pathTable)
{
Contract.Requires(startDir.IsValid);
for (; startDir != AbsolutePath.Invalid; startDir = startDir.GetParent(pathTable))
{
var configFilename = startDir.Combine(pathTable, Names.ConfigBc);
var legacyConfigFilename = startDir.Combine(pathTable, Names.ConfigDsc);
if (engine.FileExists(legacyConfigFilename))
{
return(legacyConfigFilename);
}
if (engine.FileExists(configFilename))
{
return(configFilename);
}
}
return(AbsolutePath.Invalid);
}
public static TestEnv CreateTestEnvWithPausedScheduler(string testName, List <IMount> mounts = null, PathTable pathTable = null, bool enableLazyOutputMaterialization = false)
{
return(new TestEnv(testName, FakeTestRoot, mounts: mounts, pathTable: pathTable, enableLazyOutputMaterialization: enableLazyOutputMaterialization));
}
public TestEnv(
string name,
string rootPath,
bool enableLazyOutputMaterialization = false,
int maxRelativeOutputDirectoryLength = 260,
List <IMount> mounts = null,
PathTable pathTable = null)
{
Contract.Requires(name != null);
Contract.Requires(!string.IsNullOrEmpty(rootPath));
LoggingContext = new LoggingContext("TestLogger." + name);
PathTable = pathTable ?? new PathTable();
PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });
// The tests that use TestEnv need to be modernized to take a filesystem
var fileSystem = new PassThroughFileSystem(PathTable);
Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);
// Add some well-known paths with fixed casing to the Context.PathTable
AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
var root = AbsolutePath.Create(Context.PathTable, rootPath);
var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);
configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
configuration.Engine.MaxRelativeOutputDirectoryLength = maxRelativeOutputDirectoryLength;
configuration.Schedule.EnableLazyOutputMaterialization = enableLazyOutputMaterialization;
configuration.Schedule.UnsafeDisableGraphPostValidation = false;
configuration.Schedule.ComputePipStaticFingerprints = true;
configuration.Sandbox.FileAccessIgnoreCodeCoverage = true;
BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);
Configuration = configuration;
var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);
if (mounts != null)
{
foreach (var mount in mounts)
{
mountsTable.AddResolvedMount(mount);
}
}
AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");
var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);
PipTable = graph.PipTable;
PipGraph = graph;
var locationData = new LocationData(specFile, 0, 0);
PipGraph.AddModule(ModulePip.CreateForTesting(Context.StringTable, specFile));
PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, new ModuleId(0)));
PipConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
ObjectRoot,
redirectedRoot: Configuration.Layout.RedirectedDirectory,
pipGraph: PipGraph,
moduleName: "TestModule",
symbol: name,
specPath: specFile);
Paths = new Paths(PathTable);
mountsTable.CompleteInitialization();
}
public void SerializeSandboxedProcessInfo(bool useNullFileStorage)
{
var pt = new PathTable();
var fam =
new FileAccessManifest(pt, CreateDirectoryTranslator())
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false
};
var vac = new ValidationDataCreator(fam, pt);
vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
SandboxedProcessStandardFiles standardFiles = null;
ISandboxedProcessFileStorage fileStorage;
if (useNullFileStorage)
{
fileStorage = null;
}
else
{
standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
fileStorage = new StandardFileStorage(standardFiles);
}
var envVars = new Dictionary <string, string>()
{
["Var1"] = "Val1",
["Var2"] = "Val2",
};
IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
var sidebandLogFile = A("C", "engine-cache", "sideband-logs", "log-1");
var loggerRootDirs = new[] { A("C", "out", "dir1"), A("C", "out", "dir2") };
var sharedOpaqueOutputLogger = new SidebandWriter(DefaultSidebandMetadata, sidebandLogFile, loggerRootDirs);
SandboxedProcessInfo info = new SandboxedProcessInfo(
pt,
fileStorage,
A("C", "tool", "tool.exe"),
fam,
true,
null,
sidebandWriter: sharedOpaqueOutputLogger)
{
Arguments = @"/arg1:val1 /arg2:val2",
WorkingDirectory = A("C", "Source"),
EnvironmentVariables = buildParameters,
Timeout = TimeSpan.FromMinutes(15),
PipSemiStableHash = 0x12345678,
PipDescription = nameof(SerializeSandboxedProcessInfo),
TimeoutDumpDirectory = A("C", "Timeout"),
SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
StandardObserverDescriptor = new SandboxObserverDescriptor()
{
WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
},
};
// Serialize and deserialize.
SandboxedProcessInfo readInfo = null;
using (var stream = new MemoryStream())
{
info.Serialize(stream);
stream.Position = 0;
readInfo = SandboxedProcessInfo.Deserialize(
stream,
new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
null);
}
using (readInfo.SidebandWriter)
{
// Verify.
XAssert.AreEqual(info.FileName, readInfo.FileName);
XAssert.AreEqual(info.Arguments, readInfo.Arguments);
XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
XAssert.AreEqual(envVars.Count, readEnvVars.Count);
foreach (var kvp in envVars)
{
XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
}
XAssert.AreEqual(info.Timeout, readInfo.Timeout);
XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
{
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
}
XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
if (useNullFileStorage)
{
XAssert.IsNull(readInfo.SandboxedProcessStandardFiles);
XAssert.IsNull(readInfo.FileStorage);
}
else
{
XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
}
XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
XAssert.AreEqual(sidebandLogFile, readInfo.SidebandWriter.SidebandLogFile);
XAssert.ArrayEqual(loggerRootDirs, readInfo.SidebandWriter.RootDirectories.ToArray());
ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
}
}
public static TestEnv CreateTestEnvWithPausedScheduler(List <IMount> mounts = null, PathTable pathTable = null)
{
return(CreateTestEnvWithPausedScheduler(GetTestNameGuess(), mounts, pathTable));
}
protected BaseEngineTest(ITestOutputHelper output)
: base(output)
{
TestOutput = output;
m_ignoreWarnings = OperatingSystemHelper.IsUnixOS; // ignoring /bin/sh is being used as a source file
RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
RegisterEventSource(global::BuildXL.Pips.ETWLogger.Log);
RegisterEventSource(global::BuildXL.FrontEnd.Script.ETWLogger.Log);
RegisterEventSource(global::BuildXL.FrontEnd.Core.ETWLogger.Log);
RegisterEventSource(global::BuildXL.Engine.ETWLogger.Log);
RegisterEventSource(global::BuildXL.Processes.ETWLogger.Log);
ParseAndEvaluateLogger = Logger.CreateLoggerWithTracking();
InitializationLogger = InitializationLogger.CreateLogger();
var pathTable = new PathTable();
FileSystem = new PassThroughMutableFileSystem(pathTable);
Context = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
MainSourceResolverModules = new List <AbsolutePath>();
var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");
Configuration = new CommandLineConfiguration()
{
DisableDefaultSourceResolver = true,
Resolvers = new List <IResolverSettings>
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = MainSourceResolverModules,
},
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = new List <AbsolutePath>
{
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc")),
},
},
},
Layout =
{
SourceDirectory = rootPath,
OutputDirectory = Combine(rootPath, "out"),
ObjectDirectory = Combine(rootPath, "obj"),
CacheDirectory = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled,
CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable, "cache.log")),
},
Engine =
{
ReuseEngineState = false,
LogStatistics = false,
TrackBuildsInUserFolder = false,
},
FrontEnd =
{
MaxFrontEndConcurrency = 1,
LogStatistics = false,
},
Schedule =
{
MaxIO = 1,
MaxProcesses = 1,
},
Sandbox =
{
FileSystemMode = FileSystemMode.RealAndMinimalPipGraph,
OutputReportingMode = OutputReportingMode.FullOutputOnError,
},
Logging =
{
LogsDirectory = logsPath,
LogStats = false,
LogExecution = false,
LogCounters = false,
LogMemory = false,
StoreFingerprints = false,
NoWarnings =
{
909, // Disable warnings about experimental feature
},
}
};
if (TryGetSubstSourceAndTarget(out string substSource, out string substTarget))
{
// Directory translation is needed here particularly when the test temporary directory
// is inside a directory that is actually a junction to another place.
// For example, the temporary directory is D:\src\BuildXL\Out\Object\abc\t_1, but
// the path D:\src\BuildXL\Out or D:\src\BuildXL\Out\Object is a junction to K:\Out.
// Some tool, like cmd, can access the path in K:\Out, and thus the test will have a DFA
// if there's no directory translation.
// This problem does not occur when only substs are involved, but no junctions. The method
// TryGetSubstSourceAndTarget works to get translations due to substs or junctions.
AbsolutePath substSourcePath = AbsolutePath.Create(Context.PathTable, substSource);
AbsolutePath substTargetPath = AbsolutePath.Create(Context.PathTable, substTarget);
Configuration.Engine.DirectoriesToTranslate.Add(
new TranslateDirectoryData(I($"{substSource}<{substTarget}"), substSourcePath, substTargetPath));
}
}
private static bool IsWellKnownConfigurationFileExists(AbsolutePath directory, PathTable pathTable, IFileSystem fileSystem)
{
foreach (var path in Names.WellKnownConfigFileNames)
{
var absolutePath = directory.Combine(pathTable, path);
if (fileSystem.Exists(absolutePath))
{
return(true);
}
}
return(false);
}
/// <summary>
/// Whether <param name="filePath"/> (created with <param name="pathTable"/> looks like a DScript project file.
/// </summary>
public static bool IsPathToProjectFile(AbsolutePath filePath, PathTable pathTable)
{
var fileName = filePath.GetName(pathTable).ToString(pathTable.StringTable);
return(ExtensionUtilities.IsNonConfigurationFile(fileName));
}
/// <summary>
/// When a corruption is detected, moves the engine state into the logs directory for future debugging
/// and to prevent the corrupt state from impacting future builds.
/// </summary>
public static bool TryLogAndRemoveCorruptEngineState(IConfiguration configuration, PathTable pathTable, LoggingContext loggingContext)
{
var engineCache = configuration.Layout.EngineCacheDirectory.ToString(pathTable);
var fileContentTableFile = configuration.Layout.FileContentTableFile.ToString(pathTable);
bool success = true;
// Non-recursive directory enumeration to prevent deleting folders which are persisted build-over-build in the engine cache
// but are not engine state (ex: HistoricMetadatCache and FingerprintStore)
FileUtilities.EnumerateDirectoryEntries(engineCache, (fileName, attributes) =>
{
if (!FileUtilities.IsDirectoryNoFollow(attributes))
{
var filePath = Path.Combine(engineCache, fileName);
success &= SchedulerUtilities.TryLogAndMaybeRemoveCorruptFile(
filePath,
configuration,
pathTable,
loggingContext,
removeFile: filePath != fileContentTableFile /* exclude the file content table which can impact performance significantly if deleted */);
}
});
return(success);
}
/// <nodoc />
public void Serialize(
PathTable pathTable,
BuildXLWriter writer,
PathExpander pathExpander = null,
Action <BuildXLWriter, AbsolutePath> pathWriter = null,
Action <BuildXLWriter, StringId> stringWriter = null)
{
// We allow duplicates on construction (and deserialization), but attempt to remove them here.
// This isn't required for correctness, but may result in storing fewer PathSets.
int countWithoutDuplicates = Paths.Length == 0 ? 0 : 1;
for (int i = 1; i < Paths.Length; i++)
{
if (Paths[i - 1].Path != Paths[i].Path)
{
countWithoutDuplicates++;
}
}
writer.WriteCompact(countWithoutDuplicates);
AbsolutePath last = AbsolutePath.Invalid;
string lastExpanded = null;
foreach (ObservedPathEntry entry in Paths)
{
if (last == entry.Path)
{
continue;
}
writer.Write((byte)entry.Flags);
if (entry.DirectoryEnumerationWithCustomPattern)
{
writer.Write(entry.EnumeratePatternRegex);
}
if (pathWriter != null)
{
pathWriter(writer, entry.Path);
}
else
{
// Try to tokenize the path if the pathExpander is given.
string expanded = pathExpander?.ExpandPath(pathTable, entry.Path) ?? entry.Path.ToString(pathTable);
if (!OperatingSystemHelper.IsUnixOS)
{
expanded = expanded.ToUpperInvariant();
}
int reuseCount = 0;
if (lastExpanded != null)
{
int limit = Math.Min(lastExpanded.Length, expanded.Length);
for (; reuseCount < limit && expanded[reuseCount] == lastExpanded[reuseCount]; reuseCount++)
{
;
}
}
if (OperatingSystemHelper.IsUnixOS && reuseCount == 1)
{
// As the root is denoted via '/' but the same symbol is also used as path separator,
// we cannot reuse the root path only when paths differ from the 2nd char onwards.
reuseCount = 0;
}
writer.WriteCompact(reuseCount);
writer.Write(reuseCount == 0 ? expanded : expanded.Substring(reuseCount));
lastExpanded = expanded;
}
last = entry.Path;
}
// Serializing observedAccessedFileNames
int fileNameCountWithoutDuplicates = ObservedAccessedFileNames.Length == 0 ? 0 : 1;
for (int i = 1; i < ObservedAccessedFileNames.Length; i++)
{
if (ObservedAccessedFileNames[i - 1] != ObservedAccessedFileNames[i])
{
fileNameCountWithoutDuplicates++;
}
}
writer.WriteCompact(fileNameCountWithoutDuplicates);
StringId lastFileName = StringId.Invalid;
foreach (var entry in ObservedAccessedFileNames)
{
if (lastFileName == entry)
{
continue;
}
if (stringWriter != null)
{
stringWriter(writer, entry);
}
else
{
writer.Write(entry.ToString(pathTable.StringTable));
}
}
// Serializing UnsafeOptions
UnsafeOptions.Serialize(writer);
}
protected override FrontEndContext CreateFrontEndContext(PathTable pathTable, IFileSystem fileSystem)
{
return(FrontEndContext.CreateInstanceForTesting(pathTable: pathTable, fileSystem: fileSystem, cancellationToken: TokenSource.Token));
}
/// <nodoc />
public static Possible <ObservedPathSet, DeserializeFailure> TryDeserialize(
PathTable pathTable,
BuildXLReader reader,
PathExpander pathExpander = null,
Func <BuildXLReader, AbsolutePath> pathReader = null,
Func <BuildXLReader, StringId> stringReader = null)
{
PathTable.ExpandedAbsolutePathComparer comparer = pathTable.ExpandedPathComparer;
int pathCount = reader.ReadInt32Compact();
ObservedPathEntry[] paths = new ObservedPathEntry[pathCount];
string lastStr = null;
AbsolutePath lastPath = default(AbsolutePath);
for (int i = 0; i < pathCount; i++)
{
var flags = (ObservedPathEntryFlags)reader.ReadByte();
string enumeratePatternRegex = null;
if ((flags & ObservedPathEntryFlags.DirectoryEnumerationWithCustomPattern) != 0)
{
enumeratePatternRegex = reader.ReadString();
}
else if ((flags & ObservedPathEntryFlags.DirectoryEnumerationWithAllPattern) != 0)
{
enumeratePatternRegex = RegexDirectoryMembershipFilter.AllowAllRegex;
}
AbsolutePath newPath;
string full = null;
if (pathReader != null)
{
newPath = pathReader(reader);
}
else
{
int reuseCount = reader.ReadInt32Compact();
if (reuseCount == 0)
{
full = reader.ReadString();
}
else
{
if (lastStr == null || lastStr.Length < reuseCount)
{
// This path set is invalid.
return(new DeserializeFailure($"Invalid reuseCount: {reuseCount}; last: '{lastStr}', last string length: {lastStr?.Length}"));
}
string partial = reader.ReadString();
full = lastStr.Substring(0, reuseCount) + partial;
}
if (!AbsolutePath.TryCreate(pathTable, full, out newPath))
{
// It might be failed due to the tokenized path.
if (pathExpander == null || !pathExpander.TryCreatePath(pathTable, full, out newPath))
{
return(new DeserializeFailure($"Invalid path: '{full}'"));
}
}
}
paths[i] = new ObservedPathEntry(newPath, flags, enumeratePatternRegex);
if (lastPath.IsValid)
{
#if DEBUG
if (comparer.Compare(lastPath, newPath) >= 0)
{
return(new DeserializeFailure($"Paths not sorted: " +
$"old = '{lastPath.ToString(pathTable)}', new = '{newPath.ToString(pathTable)}';" +
$"old str = '{lastStr}', new str = '{full}'"));
}
#endif
}
lastPath = newPath;
lastStr = full;
}
int fileNameCount = reader.ReadInt32Compact();
StringId[] fileNames = new StringId[fileNameCount];
for (int i = 0; i < fileNameCount; i++)
{
fileNames[i] = stringReader?.Invoke(reader) ?? StringId.Create(pathTable.StringTable, reader.ReadString());
}
// Read unsafe options
var unsafeOptions = UnsafeOptions.TryDeserialize(reader);
if (unsafeOptions == null)
{
return(new DeserializeFailure("UnsafeOptions are null"));
}
// Note that we validated sort order above.
return(new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(comparer)),
SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .FromWithoutCopy(fileNames),
new CaseInsensitiveStringIdComparer(pathTable.StringTable)),
unsafeOptions));
}
/// <summary>
/// TODO: This is code duplicated from the configuration processor. Consider refactoring
/// </summary>
private static AbsolutePath FindPrimaryConfiguration(AbsolutePath rootFolder, FrontEndEngineAbstraction engine, PathTable pathTable)
{
return(TryFindConfig(rootFolder, engine, pathTable));
}
/// <summary> /// Creates a container configuration info for testing purposes, where the remapping for directories can be passed explicitly as a collection of (originalDirectory, redirectedDirectory) /// </summary> public static ContainerConfiguration CreateConfigurationForTesting(PathTable pathTable, IEnumerable <(string originalDirectory, string redirectedDirectory)> directoryRemapping)
private static IReadOnlyCollection <LinterFailure> ComputeLinterFailures(Workspace workspace, BuildXL.FrontEnd.Script.Tracing.Logger logger, PathTable pathTable)
{
// Unfortunately all linter rules log using the logger directly, so we need to retrieve the event, map it to a typescript diagnostic and add it to the
// failure list
var failures = new List <LinterFailure>();
foreach (var diagnostic in logger.CapturedDiagnostics)
{
var path = diagnostic.Location != null?AbsolutePath.Create(pathTable, diagnostic.Location.Value.File) : AbsolutePath.Invalid;
if (path.IsValid && workspace.TryGetSourceFile(path, out var sourceFile))
{
INode node;
DScriptNodeUtilities.TryGetNodeAtPosition(sourceFile, diagnostic.Location.Value.GetAbsolutePosition(sourceFile), out node);
Contract.Assert(node != null);
var startPosition = Scanner.SkipOverTrivia(sourceFile, node.Pos);
var typeScriptDiagnostic = new Diagnostic(
sourceFile,
startPosition,
node.GetWidth(),
diagnostic.FullMessage,
DiagnosticCategory.Error,
diagnostic.ErrorCode);
var descriptor = workspace.GetModuleBySpecFileName(path).Descriptor;
failures.Add(new LinterFailure(descriptor, sourceFile, typeScriptDiagnostic));
}
}
return(failures);
}
/// <summary>
/// Creates a short description of the operation and path. The following is the summary for writing bar.txt and
/// reading bar2.txt:
///
/// W c:\foo\bar.txt
/// R c:\foo\bar2.txt
/// </summary>
public string ShortDescribe(PathTable pathTable)
{
return((IsWriteViolation ? WriteDescriptionPrefix : ReadDescriptionPrefix) + GetPath(pathTable));
}
/// <inheritdoc />
public AbsolutePath LogsRootDirectory(PathTable table)
{
return(LogsToRetain > 0 ? LogsDirectory.GetParent(table) : LogsDirectory);
}
/// <summary>
/// Gets the fully expanded path
/// </summary>
public string GetPath(PathTable pathTable)
{
return(Path ?? ManifestPath.ToString(pathTable));
}
/// <summary>
/// Configure the processBuilder's ResponseFile and ResponseFileData according to this specification.
/// This method mutates the processBuilder, changing its argumentsBuilder, ResponseFile and ResponseFileData accordingly.
/// We return the arguments to be passed to the process.
/// </summary>
internal PipData SplitArgumentsAndCreateResponseFileIfNeeded(ProcessBuilder processBuilder, DirectoryArtifact defaultDirectory, PathTable pathTable)
{
PipData arguments = default; // We'll return the actual arguments to be passed to the process.
var argumentsBuilder = processBuilder.ArgumentsBuilder;
var cutoffArg = m_firstArg;
// We will create a response file in the following cases:
// 1. If an explicit response file content was specified, either by having m_explicitData or
// by having m_forceCreation = true
// 2. If the argument line is too long (longer than MaxCommandLineLength) and m_firstArg is not the default.
// An additional argument is added to the process specifying the response file location, prefixed
// by m_prefix, unless m_requiresArgument was set to false.
if (!m_firstArg.IsDefault || m_explicitData.IsValid)
{
bool argumentLineTooLong = false;
if (!m_forceCreation)
{
// make a response file only if the command-line is too long
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
// Normalize choice to use response file by assuming paths are of length max path with a space. This will
// ensure there are no cases where changing the root will change whether a response file is used.
int cmdLineLength = arguments.GetMaxPossibleLength(pathTable.StringTable);
argumentLineTooLong = cmdLineLength > ProcessBuilder.MaxCommandLineLength;
}
if (m_forceCreation || argumentLineTooLong || m_explicitData.IsValid)
{
// add the explicit contents if we have to
if (m_explicitData.IsValid)
{
if (!argumentLineTooLong)
{
// If there was no overflow, we mark 'here' as the starting
// point for the response file before adding the explicit data as an 'argument'.
cutoffArg = argumentsBuilder.CreateCursor();
}
argumentsBuilder.Add(m_explicitData);
}
// create a pip data for the stuff in the response file
processBuilder.ResponseFileData = argumentsBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.CRuntimeArgumentRules, cutoffArg);
// generate the file
processBuilder.ResponseFile = FileArtifact.CreateSourceFile(GetResponseFilePath(defaultDirectory, pathTable));
argumentsBuilder.TrimEnd(cutoffArg);
processBuilder.AddUntrackedFile(processBuilder.ResponseFile);
if (m_requiresArgument)
{
if (string.IsNullOrEmpty(m_prefix))
{
argumentsBuilder.Add(processBuilder.ResponseFile);
}
else
{
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, pathTable.StringTable.Empty))
{
argumentsBuilder.Add(m_prefix);
argumentsBuilder.Add(processBuilder.ResponseFile);
}
}
}
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
}
else
{
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
return(arguments);
}
private static ICommandLineConfiguration CreateConfiguration(
IEnumerable <string> sdksToResolve,
PathTable pathTable,
AbsolutePath testFolderPath)
{
var configFilePath = testFolderPath.Combine(pathTable, Names.ConfigDsc);
var packageFilePath = testFolderPath.Combine(pathTable, Names.ModuleConfigBm);
var sdkPackages = sdksToResolve
.SelectMany(folder =>
Directory.EnumerateFiles(folder, Names.PackageConfigDsc, SearchOption.AllDirectories).Concat(
Directory.EnumerateFiles(folder, Names.ModuleConfigBm, SearchOption.AllDirectories).Concat(
Directory.EnumerateFiles(folder, Names.ModuleConfigDsc, SearchOption.AllDirectories))))
.Select(packageFile => AbsolutePath.Create(pathTable, packageFile))
.ToList();
var configuration =
new CommandLineConfiguration
{
// Have to new up the list because we have some bad semantics dealing with the list being null or not.
Packages = new List <AbsolutePath>
{
packageFilePath,
},
Resolvers =
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = sdkPackages,
},
},
FrontEnd =
{
MaxFrontEndConcurrency = 1, // Single threaded for deterministic evaluation
NameResolutionSemantics = NameResolutionSemantics.ImplicitProjectReferences,
// PreserveFullNames = true, Some comment in code as not to turn on, check with folks....
UsePartialEvaluation = true,
UseSpecPublicFacadeAndAstWhenAvailable = false,
ConstructAndSaveBindingFingerprint = false,
// Some of the DS tests fail when incremental frontend is not used
EnableIncrementalFrontEnd = true,
},
Engine =
{
TrackBuildsInUserFolder = false,
},
Layout =
{
OutputDirectory = testFolderPath.Combine(pathTable, "Out"),
ObjectDirectory = testFolderPath.Combine(pathTable, "Out").Combine(pathTable, "Objects"),
PrimaryConfigFile = configFilePath,
SourceDirectory = testFolderPath,
TempDirectory = testFolderPath.Combine(pathTable, "Out").Combine(pathTable, "Temp"),
},
Mounts =
{
},
Startup =
{
ConfigFile = configFilePath,
},
};
return(configuration);
}
/// <inheritdoc/>
IFileSystem IFileSystem.CopyWithNewPathTable(PathTable pathTable)
{
return(new EngineFileSystem(pathTable, m_engine));
}
public TestSpecInteractionStatePersistence()
{
m_pathTable = new PathTable();
m_comparer = new SpecInteractionStateEqualityComparer(m_pathTable);
}
private PageablePipStore(PathTable pathTable, SymbolTable symbolTable, PageableStore.SerializedState state, int initialBufferSize)
: base(pathTable, symbolTable, state, initialBufferSize)
{
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
FingerprintStoreTestHooks testHooks = null)
{
// Unblock caller
await Task.Yield();
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
string downLoadedPriviousFingerprintStoreSavedPath = null;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
PathTable newPathTable = new PathTable();
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable);
var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr);
var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key);
var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration.Schedule.EnvironmentFingerprint);
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(newPathTable);
downLoadedPriviousFingerprintStoreSavedPath = path;
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(
logTarget,
loggingContext,
context,
possibleStore.Result,
graph,
runnablePipPerformance,
configuration,
downLoadedPriviousFingerprintStoreSavedPath,
testHooks: testHooks));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
