public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile) { string remoteDirectoryPath = "remoteDirectory"; string fakeDirectoryId = "123:1:12345"; var directoryPath = Path.Combine(TestOutputDirectory, "foo"); if (!Directory.Exists(directoryPath)) { Directory.CreateDirectory(directoryPath); } var dropPaths = new List <string>(); var dropClient = new MockDropClient(addFileFunc: (item) => { dropPaths.Add(item.RelativeDropPath); return(Task.FromResult(AddFileResult.Associated)); }); var ipcProvider = IpcFactory.GetProvider(); // this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response var ipcExecutor = new LambdaIpcOperationExecutor(op => { var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId); var file = new SealedDirectoryFile( Path.Combine(directoryPath, "file.txt"), new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1), FileContentInfo.CreateWithUnknownLength(global::BuildXL.Scheduler.WellKnownContentHashes.AbsentFile)); return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> { file }))); }); WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), (moniker, mockServer) => { var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig())); WithSetup( dropClient, (daemon, etwListener) => { var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser()); var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult(); XAssert.IsTrue(dropPaths.Count == 0); // if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded); XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode); }, bxlApiClient); return(Task.CompletedTask); }).GetAwaiter().GetResult(); }
public void TestLazilyMaterializedSymlinkRejected() { string fileId = "142342:3"; string filePath = Path.Combine(TestOutputDirectory, nameof(TestLazilyMaterializedSymlinkRejected) + "-test.txt"); if (File.Exists(filePath)) { File.Delete(filePath); } // this client wants to read the file var dropClient = new MockDropClient(addFileFunc: (item) => { Assert.NotNull(item.BlobIdentifier); var ex = Assert.Throws <DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult()); // rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it) throw ex; }); WithSetup(dropClient, (daemon, etwListener, dropConfig) => { var dropName = GetDropFullName(dropConfig); var ipcProvider = IpcFactory.GetProvider(); var ipcExecutor = new LambdaIpcOperationExecutor(op => { // this mock BuildXL server materializes a regular file, which we will treat as a symlink in this test var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId); File.WriteAllText(filePath, TestFileContent); return(IpcResult.Success(cmd.RenderResult(true))); }); WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), (moniker, mockServer) => { var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig())); var addFileItem = new DropItemForBuildXLFile( symlinkTester: (file) => file == filePath ? true : false, client: client, fullDropName: dropName, filePath: filePath, fileId: fileId, fileContentInfo: TestFileContentInfo); // addfile files IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult(); XAssert.IsFalse(result.Succeeded, "expected addfile to fail; instead it succeeded and returned payload: " + result.Payload); XAssert.IsTrue(result.Payload.Contains(DropItemForBuildXLFile.MaterializationResultIsSymlinkErrorPrefix)); }); }); }
public void TestAddBuildXLFile_UploadCallsBuildXLServer() { string fileId = "142342:2"; string filePath = Path.Combine(TestOutputDirectory, nameof(TestAddBuildXLFile_UploadCallsBuildXLServer) + "-test.txt"); if (File.Exists(filePath)) { File.Delete(filePath); } // this client wants to read the file var dropClient = new MockDropClient(addFileFunc: (item) => { Assert.NotNull(item.BlobIdentifier); var fileInfo = item.EnsureMaterialized().GetAwaiter().GetResult(); XAssert.IsTrue(fileInfo != null && fileInfo.Exists); XAssert.AreEqual(TestFileContent, File.ReadAllText(fileInfo.FullName)); return(Task.FromResult(AddFileResult.UploadedAndAssociated)); }); WithSetup(dropClient, (daemon, etwListener, dropConfig) => { var dropName = GetDropFullName(dropConfig); var ipcProvider = IpcFactory.GetProvider(); var ipcExecutor = new LambdaIpcOperationExecutor(op => { var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId); File.WriteAllText(filePath, TestFileContent); return(IpcResult.Success(cmd.RenderResult(true))); }); WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), (moniker, mockServer) => { var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig())); var addFileItem = new DropItemForBuildXLFile(client, dropName, filePath, fileId, fileContentInfo: TestFileContentInfo); // addfile succeeds IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult(); XAssert.IsTrue(result.Succeeded, result.Payload); }); }); }
public async Task TestCommandThatRequestsStopReceivesResponseBeforeDisconnectResponse() { const string StopCmd = "stop"; const string StopResponse = "stopped"; IServer theServer = null; // set at the beginning of testAction var ipcExecutor = new LambdaIpcOperationExecutor(op => { Assert.Equal(StopCmd, op.Payload); Assert.NotNull(theServer); theServer.RequestStop(); return(IpcResult.Success(StopResponse)); }); await WithServerAndClient( nameof(TestCommandThatRequestsStopReceivesResponseBeforeDisconnectResponse), ipcExecutor, maxConcurrentClients : 1, maxConcurrentRequestsPerClient : 1, testAction : async(server, tcpProvider, client) => { theServer = server; // send stop request var stopRequest = new Request(new IpcOperation(StopCmd, waitForServerAck: true)); await stopRequest.SerializeAsync(client); // assert response to the stop request is received first var resp = await Response.DeserializeAsync(client); Assert.Equal(stopRequest.Id, resp.RequestId); Assert.True(resp.Result.Succeeded, "Expected response to succeed; error: " + resp.Result.Payload); Assert.Equal(StopResponse, resp.Result.Payload); // assert disconnect response is received next resp = await Response.DeserializeAsync(client); Assert.True(resp.IsDisconnectResponse, "Expected a disconnect response, got: " + resp); }); }
public async Task TestMaxConcurrentRequestPerClient() { var mre = new ManualResetEvent(false); var waitReq = new Request(new IpcOperation("wait", waitForServerAck: true)); var signalReq = new Request(new IpcOperation("signal", waitForServerAck: true)); var waitSignalExecutor = new LambdaIpcOperationExecutor(op => { if (op.Payload == waitReq.Operation.Payload) { mre.WaitOne(); return(IpcResult.Success()); } if (op.Payload == signalReq.Operation.Payload) { mre.Set(); return(IpcResult.Success()); } Assert.False(true); return(null); }); // with 2 max concurrent requests await WithServerAndClient( nameof(TestMaxConcurrentRequestPerClient) + ".2", waitSignalExecutor, maxConcurrentRequestsPerClient : 2, testAction : async(server, tcpProvider, clientStream) => { // send "wait" first, then "signal" await waitReq.SerializeAsync(clientStream); await signalReq.SerializeAsync(clientStream); // both responses arrive await Response.DeserializeAsync(clientStream); await Response.DeserializeAsync(clientStream); }); mre.Reset(); // with 1 max concurrent requests await WithServerAndClient( nameof(TestMaxConcurrentRequestPerClient) + ".1", waitSignalExecutor, maxConcurrentClients : 2, maxConcurrentRequestsPerClient : 1, testAction : async(server, tcpProvider, client) => { // send "wait" first, then "signal" await waitReq.SerializeAsync(client); await signalReq.SerializeAsync(client); // send signal from a new client using (var tcpClient2 = await tcpProvider.ConnectToServerAsync()) using (var clientStream2 = tcpClient2.GetStream()) { await signalReq.SerializeAsync(clientStream2); var resp = await Response.DeserializeAsync(clientStream2); Assert.True(resp.Result.Succeeded); } // finally both client1 tasks complete var resp1 = await Response.DeserializeAsync(client); var resp2 = await Response.DeserializeAsync(client); Assert.True(resp1.Result.Succeeded); Assert.True(resp2.Result.Succeeded); }); }
[InlineData(".*d\\.txt$")] // no files public void TestAddDirectoryToDropWithFilters(string filter) { // TestOutputDirectory // |- foo <directory> <- 'uploading' this directory // |- a.txt // |- b.txt // |- bar <directory> // |- c.txt string remoteDirectoryPath = "remoteDirectory"; string fakeDirectoryId = "123:1:12345"; var directoryPath = Path.Combine(TestOutputDirectory, "foo"); var files = new List <(string fileName, string remoteFileName)> { (Path.Combine(directoryPath, "a.txt"), $"{remoteDirectoryPath}/a.txt"), (Path.Combine(directoryPath, "b.txt"), $"{remoteDirectoryPath}/b.txt"), (Path.Combine(directoryPath, "bar", "c.txt"), $"{remoteDirectoryPath}/bar/c.txt"), }; var dropPaths = new List <string>(); var expectedDropPaths = new HashSet <string>(); var regex = new Regex(filter, RegexOptions.IgnoreCase | RegexOptions.Compiled); expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName)); if (!Directory.Exists(directoryPath)) { Directory.CreateDirectory(directoryPath); } var dropClient = new MockDropClient(addFileFunc: (item) => { dropPaths.Add(item.RelativeDropPath); return(Task.FromResult(AddFileResult.UploadedAndAssociated)); }); var ipcProvider = IpcFactory.GetProvider(); // this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response var ipcExecutor = new LambdaIpcOperationExecutor(op => { var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId); // Now 'fake' the response - here we only care about the 'FileName' field. // In real life it's not the case, but this is a test and our custom addFileFunc // in dropClient simply collects the drop file names. var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList(); return(IpcResult.Success(cmd.RenderResult(result))); }); WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), (moniker, mockServer) => { var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig())); WithSetup( dropClient, (daemon, etwListener) => { var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs( $"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}", new UnixParser()); var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult(); XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload); XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true); }, bxlApiClient); return(Task.CompletedTask); }).GetAwaiter().GetResult(); }
public void TestAddDirectoryToDrop() { var dropPaths = new System.Collections.Generic.List <string>(); var expectedDropPaths = new System.Collections.Generic.HashSet <string>(); string directoryId = "123:1:12345"; // TestOutputDirectory // |- foo <directory> <- 'uploading' this directory // |- b.txt // |- c.txt // |- bar <directory> // |- d.txt var path = Path.Combine(TestOutputDirectory, "foo"); if (Directory.Exists(path)) { Directory.Delete(path, recursive: true); } Directory.CreateDirectory(path); var fileFooB = Path.Combine(TestOutputDirectory, "foo", "b.txt"); File.WriteAllText(fileFooB, Guid.NewGuid().ToString()); expectedDropPaths.Add("remote/b.txt"); var fileFooC = Path.Combine(TestOutputDirectory, "foo", "c.txt"); File.WriteAllText(fileFooC, Guid.NewGuid().ToString()); expectedDropPaths.Add("remote/c.txt"); path = Path.Combine(TestOutputDirectory, "foo", "bar"); Directory.CreateDirectory(path); var fileFooBarD = Path.Combine(TestOutputDirectory, "foo", "bar", "d.txt"); File.WriteAllText(fileFooBarD, Guid.NewGuid().ToString()); expectedDropPaths.Add("remote/bar/d.txt"); var dropClient = new MockDropClient(addFileFunc: (item) => { dropPaths.Add(item.RelativeDropPath); return(Task.FromResult(AddFileResult.UploadedAndAssociated)); }); var ipcProvider = IpcFactory.GetProvider(); var ipcExecutor = new LambdaIpcOperationExecutor(op => { // this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, directoryId); // Now 'fake' the response - here we only care about the 'FileName' field. // In real life it's not the case, but this is a test and our custom addFileFunc // in dropClient simply collects the drop file names. var result = new System.Collections.Generic.List <SealedDirectoryFile> { CreateFakeSealedDirectoryFile(fileFooB), CreateFakeSealedDirectoryFile(fileFooC), CreateFakeSealedDirectoryFile(fileFooBarD) }; return(IpcResult.Success(cmd.RenderResult(result))); }); WithSetup(dropClient, async(daemon, etwListener) => { await WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), async(moniker, mockServer) => { var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig())); var ipcResult = await daemon.AddDirectoryAsync(Path.Combine(TestOutputDirectory, "foo"), directoryId, "remote", TestChunkDedup, client); XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload); XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true); }); }); }
public void TestAdddingDirectoryToDropWithSpecifiedRelativePathReplacement(string dropPath, string replaceOldValue, string replaceNewValue, params string[] expectedFiles) { /* * Directory content: * a * dir\b * dir\dir\c * dir\dir2\d * dir3\e */ var expectedDropPaths = new HashSet <string>(expectedFiles); var dropPaths = new List <string>(); var fakeDirectoryId = "123:1:12345"; var directoryPath = Path.Combine(TestOutputDirectory, "foo"); var files = new List <string> { Path.Combine(directoryPath, "a"), Path.Combine(directoryPath, "dir", "b"), Path.Combine(directoryPath, "dir", "dir", "c"), Path.Combine(directoryPath, "dir", "dir2", "d"), Path.Combine(directoryPath, "dir3", "e"), }; var dropClient = new MockDropClient(addFileFunc: (item) => { dropPaths.Add(item.RelativeDropPath); return(Task.FromResult(AddFileResult.UploadedAndAssociated)); }); var ipcProvider = IpcFactory.GetProvider(); // this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response var ipcExecutor = new LambdaIpcOperationExecutor(op => { var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId); // Now 'fake' the response - here we only care about the 'FileName' field. // In real life it's not the case, but this is a test and our custom addFileFunc // in dropClient simply collects the drop file names. var result = files.Select(file => CreateFakeSealedDirectoryFile(file)).ToList(); return(IpcResult.Success(cmd.RenderResult(result))); }); WithIpcServer( ipcProvider, ipcExecutor, new ServerConfig(), (moniker, mockServer) => { var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker); WithSetup( dropClient, (daemon, etwListener, dropConfig) => { var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs( $"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {dropPath} --directoryFilter .* --directoryRelativePathReplace {serializeReplaceArgument(replaceOldValue, replaceNewValue)} --directoryFilterUseRelativePath false", new UnixParser()); var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult(); XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload); XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true); }, bxlApiClient); return(Task.CompletedTask); }).GetAwaiter().GetResult(); string serializeReplaceArgument(string oldValue, string newValue) { if (oldValue != null || newValue != null) { return($"#{oldValue}#{newValue}#"); } return("##"); } }