public void ValidateSharedCompilationAccessesAgainstNonSharedCompilation()
{
// Run the same managed csc call with and without shared compilation
RunManagedCompilation(useSharedCompilation: true, out _, out var allSharedInputs, out var allSharedOutputs);
RunManagedCompilation(useSharedCompilation: false, out _, out var allNonSharedInputs, out var allNonSharedOutputs);
// All inputs and outputs should match
XAssert.AreSetsEqual(allSharedInputs.ToHashSet(), allNonSharedInputs.ToHashSet(), expectedResult: true);
XAssert.AreSetsEqual(allSharedOutputs.ToHashSet(), allNonSharedOutputs.ToHashSet(), expectedResult: true);
}
public void TestFilterDirectoryContent()
{
const string DirPath = @"c:\a";
var files = new List <SealedDirectoryFile>
{
new SealedDirectoryFile(@"c:\a\1.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
new SealedDirectoryFile(@"c:\a\dir\foo.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
new SealedDirectoryFile(@"c:\a\foo.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
};
var regex = new Regex(@".*\.txt", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
var expectedResult = new List <SealedDirectoryFile>
{
new SealedDirectoryFile(@"c:\a\1.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
new SealedDirectoryFile(@"c:\a\dir\foo.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
new SealedDirectoryFile(@"c:\a\foo.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
};
var result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: false);
XAssert.AreSetsEqual(expectedResult, result, true);
result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: true);
XAssert.AreSetsEqual(expectedResult, result, true);
regex = new Regex(@"c:\\a\\1\.txt", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
expectedResult = new List <SealedDirectoryFile>
{
new SealedDirectoryFile(@"c:\a\1.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
};
result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: false);
XAssert.AreSetsEqual(expectedResult, result, true);
result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: true);
XAssert.AreSetsEqual(expectedResult, result, false);
regex = new Regex(@"\Gfoo.txt", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
expectedResult = new List <SealedDirectoryFile>
{
new SealedDirectoryFile(@"c:\a\foo.txt", FileArtifact.Invalid, FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.EmptyHash)),
};
result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: false);
XAssert.AreSetsEqual(expectedResult, result, false);
result = global::Tool.DropDaemon.DropDaemon.FilterDirectoryContent(DirPath, files, regex, applyFilterToRelativePath: true);
XAssert.AreSetsEqual(expectedResult, result, true);
}
public void TestBigGraphWithConcurrentTypeChecking()
{
var random = new Random();
// create a random big graph representing spec dependencies
var numNodes = 500;
var dependencyGraph = CreateRandomBigGraph(random, numNodes);
// construct spec files according to generated spec dependencies
var specs = dependencyGraph
.Nodes
.Select(i => GenerateSpecContent(i, dependencyGraph))
.ToList();
var randomSpecIdx = random.Next(dependencyGraph.NodeCount);
var expectedDownstream = dependencyGraph.OutgoingEdges(randomSpecIdx).Select(e => AbsolutePath.Create(m_pathTable, GetSpecName(e.Dest)));
var expectedUpstream = dependencyGraph.IncomingEdges(randomSpecIdx).Select(e => AbsolutePath.Create(m_pathTable, GetSpecName(e.Src)));
// create a checker and add generated files to it
var checker = dependencyGraph.Nodes.Aggregate(
CheckerWithPrelude(),
(checkerAcc, specIdx) => checkerAcc.AddSourceFileToDefaultModule(sourceFileContent: specs[specIdx], fileName: GetSpecName(specIdx)));
// type check
var diagnostics = checker.RunChecker(trackFileToFileDependencies: true, degreeOfParallelism: Math.Max(Environment.ProcessorCount, 5));
XAssert.AreEqual(0, diagnostics.Count, "Expected 0 diagnostics, got: " + string.Join(Environment.NewLine, diagnostics.Select(d => d.ToString())));
// get computed dependencies
var sourceFile = checker.GetSourceFile(GetSpecName(randomSpecIdx));
var asf = new AnalyzedSourceFile(checker.TypeChecker, sourceFile, checker.GetSourceFiles(), m_pathTable);
var downstream = asf.DownStreamDependencies;
var upstream = asf.UpStreamDependencies;
// assert computed dependencies are correct
var messageFormat = "Dependencies do not match: node count = {0}; chosen node = {1}; graph = {2}";
var args = new object[] { numNodes, randomSpecIdx, dependencyGraph };
XAssert.AreSetsEqual(expectedDownstream, downstream, true, format: messageFormat, args: args);
XAssert.AreSetsEqual(expectedUpstream, upstream, true, format: messageFormat, args: args);
}
/// <summary>
/// Given:
/// - a file-to-file map (<paramref name="file2file"/>), and
/// - indexes of changed nodes (<paramref name="changed"/>)
/// computes
/// - downstream nodes (and asserts they match <paramref name="expectedDownstream"/>
/// - upsteram nodes (and asserts they match <paramref name="expectedUpstream"/>
/// - indirect upstream nodes (and asserts they match <paramref name="expectedIndirectUpstream"/>
/// - all affected nodes (and asserts they match <paramref name="expectedAffected"/>
///
/// Definitions:
/// - 'downstream' nodes: all nodes reachable from <paramref name="changed"/> (excluding the
/// changed nodes themselves) by following the reverse edges in <paramref name="file2file"/>
/// - 'upstream' nodes: all nodes reachable from <paramref name="changed"/> (excluding the
/// changed nodes themselves) by following the edges in <paramref name="file2file"/>
/// - 'affected' nodes: upstream of 'changed' + 'downstream' nodes, including the changed and
/// downstream nodes themselves
/// - 'indirect upstream' nodes: 'affected' nodes that are neither 'changed' or 'upstream' or 'downstream'
/// </summary>
/// <remarks>
/// Some remarks about the notation in the comments below:
/// - the '^' means transitive closure
/// - the '*' means reflexive transitive closure
/// - the '.' means relational join
/// For example
/// - changed.^Edges means all reachable nodes from 'changed' by following 'Edges', excluding the nodes in 'changed'
/// - changed.*Edges means all reachable nodes from 'changed' by following 'Edges', including the nodes in 'changed'.
/// </remarks>
private void ComputeAndCheckSpecStates(SimpleGraph file2file, int[] changed, int[] expectedDownstream,
int[] expectedUpstream, int[] expectedIndirectUpstream, int[] expectedAffected)
{
// downstream = changed.^reverseEdges
var downstream = file2file.ComputeDownstream(changed);
XAssert.AreSetsEqual(expectedDownstream, downstream, expectedResult: true, format: "Downstream nodes don't match");
// upstream = changed.^edges
var upstream = file2file.ComputeUpstream(changed);
XAssert.AreSetsEqual(expectedUpstream, upstream, expectedResult: true, format: "Upstream nodes don't match");
// allAffected = (changed.*reverseEdges).*edges
var allAffected = file2file.ComputeReflexiveUpstream(file2file.ComputeReflexiveDownstream(changed));
XAssert.AreSetsEqual(expectedAffected, allAffected, expectedResult: true, format: "Affected nodes don't match");
// indirectUpstream = allAffected - changed - upstream - downstream
var indirectUpstream = allAffected.Except(changed).Except(upstream).Except(downstream);
XAssert.AreSetsEqual(expectedIndirectUpstream, indirectUpstream, expectedResult: true, format: "Indirect upstream nodes don't match");
}
public void TestProcessExecutionResultSerialization()
{
var reportedAccess = CreateRandomReportedFileAccess();
Fingerprint fingerprint = FingerprintUtilities.CreateRandom();
var processExecutionResult = ExecutionResult.CreateSealed(
result: PipResultStatus.Succeeded,
numberOfWarnings: 12,
outputContent: ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(CreateRandomOutputContent(), CreateRandomOutputContent()),
directoryOutputs: ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(CreateRandomOutputDirectory(), CreateRandomOutputDirectory()),
performanceInformation: new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
DateTime.UtcNow,
DateTime.UtcNow + TimeSpan.FromMinutes(2),
FingerprintUtilities.ZeroFingerprint,
TimeSpan.FromMinutes(2),
new FileMonitoringViolationCounters(2, 3, 4),
default(IOCounters),
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(3),
ProcessMemoryCounters.CreateFromBytes(12324, 12325, 12326, 12326),
33,
7,
0),
fingerprint: new WeakContentFingerprint(fingerprint),
fileAccessViolationsNotAllowlisted: new[]
{
reportedAccess,
CreateRandomReportedFileAccess(),
// Create reported file access that uses the same process to test deduplication during deserialization
CreateRandomReportedFileAccess(reportedAccess.Process),
},
allowlistedFileAccessViolations: new ReportedFileAccess[0],
mustBeConsideredPerpetuallyDirty: true,
dynamicallyObservedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyProbedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyObservedEnumerations: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path
),
allowedUndeclaredSourceReads: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
absentPathProbesUnderOutputDirectories: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
twoPhaseCachingInfo: new TwoPhaseCachingInfo(
new WeakContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
ContentHashingUtilities.CreateRandom(),
new StrongContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
new CacheEntry(ContentHashingUtilities.CreateRandom(), null, CreateRandomContentHashArray())),
pipCacheDescriptorV2Metadata: null,
converged: true,
pathSet: null,
cacheLookupStepDurations: null,
pipProperties: new Dictionary <string, int> {
{ "Foo", 1 }, { "Bar", 9 }
},
hasUserRetries: true);
ExecutionResultSerializer serializer = new ExecutionResultSerializer(0, Context);
ExecutionResult deserializedProcessExecutionResult;
using (var stream = new MemoryStream())
using (var writer = new BuildXLWriter(false, stream, true, false))
using (var reader = new BuildXLReader(false, stream, true))
{
serializer.Serialize(writer, processExecutionResult, preservePathCasing: false);
stream.Position = 0;
deserializedProcessExecutionResult = serializer.Deserialize(reader,
processExecutionResult.PerformanceInformation.WorkerId);
}
// Ensure successful pip result is changed to not materialized.
XAssert.AreEqual(PipResultStatus.NotMaterialized, deserializedProcessExecutionResult.Result);
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.NumberOfWarnings,
r => r.Converged,
r => r.OutputContent.Length,
r => r.DirectoryOutputs.Length,
r => r.PerformanceInformation.ExecutionLevel,
r => r.PerformanceInformation.ExecutionStop,
r => r.PerformanceInformation.ExecutionStart,
r => r.PerformanceInformation.ProcessExecutionTime,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessViolationsNotAllowlisted,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedAndCacheable,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedButNotCacheable,
r => r.PerformanceInformation.UserTime,
r => r.PerformanceInformation.KernelTime,
r => r.PerformanceInformation.MemoryCounters.PeakWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.AverageWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.PeakCommitSizeMb,
r => r.PerformanceInformation.MemoryCounters.AverageCommitSizeMb,
r => r.PerformanceInformation.NumberOfProcesses,
r => r.FileAccessViolationsNotAllowlisted.Count,
r => r.MustBeConsideredPerpetuallyDirty,
r => r.DynamicallyObservedFiles.Length,
r => r.DynamicallyProbedFiles.Length,
r => r.DynamicallyObservedEnumerations.Length,
r => r.AllowedUndeclaredReads.Count,
r => r.TwoPhaseCachingInfo.WeakFingerprint,
r => r.TwoPhaseCachingInfo.StrongFingerprint,
r => r.TwoPhaseCachingInfo.PathSetHash,
r => r.TwoPhaseCachingInfo.CacheEntry.MetadataHash,
r => r.TwoPhaseCachingInfo.CacheEntry.OriginatingCache,
r => r.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length,
r => r.PipProperties.Count,
r => r.HasUserRetries,
r => r.RetryInfo
);
for (int i = 0; i < processExecutionResult.OutputContent.Length; i++)
{
int j = i;
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.OutputContent[j].Item1,
r => r.OutputContent[j].Item2
);
// Ensure that output origin from deserialzed output content is changed to not materialized.
XAssert.AreEqual(PipOutputOrigin.NotMaterialized, deserializedProcessExecutionResult.OutputContent[i].Item3);
}
for (int i = 0; i < processExecutionResult.DirectoryOutputs.Length; i++)
{
var expected = processExecutionResult.DirectoryOutputs[i];
var result = deserializedProcessExecutionResult.DirectoryOutputs[i];
XAssert.AreEqual(expected.Item1, result.Item1);
XAssert.AreEqual(expected.Item2.Length, result.Item2.Length);
for (int j = 0; j < expected.Item2.Length; j++)
{
XAssert.AreEqual(expected.Item2[j], result.Item2[j]);
}
}
for (int i = 0; i < processExecutionResult.FileAccessViolationsNotAllowlisted.Count; i++)
{
// Compare individual fields for ReportedFileAccess since it uses reference
// equality for reported process which would not work for serialization/deserialization
AssertEqual(processExecutionResult.FileAccessViolationsNotAllowlisted[i], deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[i]);
}
// Ensure that reported process instances are deduplicated.
XAssert.AreSame(deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[0].Process,
deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[2].Process);
for (int i = 0; i < processExecutionResult.DynamicallyObservedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedFiles[i], deserializedProcessExecutionResult.DynamicallyObservedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyProbedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyProbedFiles[i], deserializedProcessExecutionResult.DynamicallyProbedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyObservedEnumerations.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedEnumerations[i], deserializedProcessExecutionResult.DynamicallyObservedEnumerations[i]);
}
XAssert.AreSetsEqual(processExecutionResult.AllowedUndeclaredReads, deserializedProcessExecutionResult.AllowedUndeclaredReads, expectedResult: true);
var referencedContentLength = processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length;
for (int i = 0; i < referencedContentLength; i++)
{
XAssert.AreEqual(
processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i],
deserializedProcessExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i]);
}
XAssert.AreEqual(9, deserializedProcessExecutionResult.PipProperties["Bar"]);
}
public void TestRedirectUserProfileDirectory()
{
// first run to create all necessary directories leading to obj directory
SetupTestData();
RunEngine();
string currentUserProfile = SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile);
string junctionPath = Path.Combine(Configuration.Layout.ObjectDirectory.ToString(Context.PathTable), "buildXLUserProfile");
bool specialFolderInitializerWasCalled = false;
var translatedDirectory = new List <TranslateDirectoryData>();
var properties = new Dictionary <string, string>();
var expectedProperties = new Dictionary <string, string>
{
{ "APPDATA", Path.Combine(junctionPath, "AppData", "Roaming") },
{ "LOCALAPPDATA", Path.Combine(junctionPath, "AppData", "Local") },
{ "USERPROFILE", junctionPath },
{ "USERNAME", "buildXLUserProfile" },
{ "HOMEDRIVE", Path.GetPathRoot(junctionPath).TrimEnd('\\') },
{ "HOMEPATH", junctionPath.Substring(Path.GetPathRoot(junctionPath).TrimEnd('\\').Length) },
{ "INTERNETCACHE", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCache") },
{ "INTERNETHISTORY", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "History") },
{ "INETCOOKIES", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCookies") },
{ "LOCALLOW", Path.Combine(junctionPath, "AppData", "LocalLow") },
};
// add the variables to the dictionary, so we can verify that the method overrides the existing values
foreach (var envVar in expectedProperties.Keys)
{
properties.Add(envVar, string.Empty);
}
try
{
var success = BuildXLEngine.RedirectUserProfileDirectory(
Configuration.Layout.ObjectDirectory,
translatedDirectory,
properties,
dict => { specialFolderInitializerWasCalled = true; },
true,
Context.PathTable,
LoggingContext);
// must have finished successfully
XAssert.IsTrue(success);
// verify the env block is properly populated
XAssert.AreSetsEqual(expectedProperties.Keys, properties.Keys, true);
XAssert.IsFalse(expectedProperties.Any(kvp => properties[kvp.Key] != kvp.Value));
XAssert.IsTrue(specialFolderInitializerWasCalled);
// verify junction
var openResult = FileUtilities.TryOpenDirectory(junctionPath, FileDesiredAccess.FileReadAttributes, FileShare.ReadWrite, FileFlagsAndAttributes.FileFlagOpenReparsePoint, out var handle);
XAssert.IsTrue(openResult.Succeeded);
using (handle)
{
var possibleTarget = FileUtilities.TryGetReparsePointTarget(handle, junctionPath);
XAssert.IsTrue(possibleTarget.Succeeded);
XAssert.AreEqual(FileSystemWin.NtPathPrefix + currentUserProfile, possibleTarget.Result);
}
// verify that we added a new directory translation
AbsolutePath.TryCreate(Context.PathTable, currentUserProfile, out var fromPath);
AbsolutePath.TryCreate(Context.PathTable, junctionPath, out var toPath);
XAssert.IsTrue(translatedDirectory.Count == 1);
XAssert.IsTrue(translatedDirectory[0].FromPath == fromPath && translatedDirectory[0].ToPath == toPath);
}
finally
{
// clean the junction after the test
var possibleProbe = FileUtilities.TryProbePathExistence(junctionPath, false);
if (possibleProbe.Succeeded && possibleProbe.Result != PathExistence.Nonexistent)
{
// we attempt to delete the junction, but we do not care if we failed to do
FileUtilities.TryRemoveDirectory(junctionPath, out int errCode);
}
}
}
[InlineData(".*d\\.txt$")] // no files
public void TestAddDirectoryToDropWithFilters(string filter)
{
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- a.txt
// |- b.txt
// |- bar <directory>
// |- c.txt
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <(string fileName, string remoteFileName)>
{
(Path.Combine(directoryPath, "a.txt"), $"{remoteDirectoryPath}/a.txt"),
(Path.Combine(directoryPath, "b.txt"), $"{remoteDirectoryPath}/b.txt"),
(Path.Combine(directoryPath, "bar", "c.txt"), $"{remoteDirectoryPath}/bar/c.txt"),
};
var dropPaths = new List <string>();
var expectedDropPaths = new HashSet <string>();
var regex = new Regex(filter, RegexOptions.IgnoreCase | RegexOptions.Compiled);
expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName));
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public bool VerifyEquals(PipExecutionDirectoryOutputs expected, PipExecutionDirectoryOutputs actual)
{
XAssert.AreEqual(expected.PipId, actual.PipId);
XAssert.AreSetsEqual(expected.DirectoryOutputs, actual.DirectoryOutputs, expectedResult: true, DirectoryOutputComparer.Instance);
return(true);
}
public void TestAddDirectoryToDrop()
{
var dropPaths = new System.Collections.Generic.List <string>();
var expectedDropPaths = new System.Collections.Generic.HashSet <string>();
string directoryId = "123:1:12345";
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- b.txt
// |- c.txt
// |- bar <directory>
// |- d.txt
var path = Path.Combine(TestOutputDirectory, "foo");
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
Directory.CreateDirectory(path);
var fileFooB = Path.Combine(TestOutputDirectory, "foo", "b.txt");
File.WriteAllText(fileFooB, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/b.txt");
var fileFooC = Path.Combine(TestOutputDirectory, "foo", "c.txt");
File.WriteAllText(fileFooC, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/c.txt");
path = Path.Combine(TestOutputDirectory, "foo", "bar");
Directory.CreateDirectory(path);
var fileFooBarD = Path.Combine(TestOutputDirectory, "foo", "bar", "d.txt");
File.WriteAllText(fileFooBarD, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/bar/d.txt");
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, directoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = new System.Collections.Generic.List <SealedDirectoryFile>
{
CreateFakeSealedDirectoryFile(fileFooB),
CreateFakeSealedDirectoryFile(fileFooC),
CreateFakeSealedDirectoryFile(fileFooBarD)
};
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithSetup(dropClient, async(daemon, etwListener) =>
{
await WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
async(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var ipcResult = await daemon.AddDirectoryAsync(Path.Combine(TestOutputDirectory, "foo"), directoryId, "remote", TestChunkDedup, client);
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
});
});
}
public void TestAdddingDirectoryToDropWithSpecifiedRelativePathReplacement(string dropPath, string replaceOldValue, string replaceNewValue, params string[] expectedFiles)
{
/*
* Directory content:
* a
* dir\b
* dir\dir\c
* dir\dir2\d
* dir3\e
*/
var expectedDropPaths = new HashSet <string>(expectedFiles);
var dropPaths = new List <string>();
var fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <string>
{
Path.Combine(directoryPath, "a"),
Path.Combine(directoryPath, "dir", "b"),
Path.Combine(directoryPath, "dir", "dir", "c"),
Path.Combine(directoryPath, "dir", "dir2", "d"),
Path.Combine(directoryPath, "dir3", "e"),
};
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(file => CreateFakeSealedDirectoryFile(file)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener, dropConfig) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {dropPath} --directoryFilter .* --directoryRelativePathReplace {serializeReplaceArgument(replaceOldValue, replaceNewValue)} --directoryFilterUseRelativePath false",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
string serializeReplaceArgument(string oldValue, string newValue)
{
if (oldValue != null || newValue != null)
{
return($"#{oldValue}#{newValue}#");
}
return("##");
}
}
