public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile)
{
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
var file = new SealedDirectoryFile(
Path.Combine(directoryPath, "file.txt"),
new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1),
FileContentInfo.CreateWithUnknownLength(global::BuildXL.Scheduler.WellKnownContentHashes.AbsentFile));
return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> {
file
})));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
// if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it
XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded);
XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public void TestLazilyMaterializedSymlinkRejected()
{
string fileId = "142342:3";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestLazilyMaterializedSymlinkRejected) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var ex = Assert.Throws <DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
// rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it)
throw ex;
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this mock BuildXL server materializes a regular file, which we will treat as a symlink in this test
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(
symlinkTester: (file) => file == filePath ? true : false,
client: client,
fullDropName: dropName,
filePath: filePath,
fileId: fileId,
fileContentInfo: TestFileContentInfo);
// addfile files
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsFalse(result.Succeeded, "expected addfile to fail; instead it succeeded and returned payload: " + result.Payload);
XAssert.IsTrue(result.Payload.Contains(DropItemForBuildXLFile.MaterializationResultIsSymlinkErrorPrefix));
});
});
}
public void TestAddBuildXLFile_UploadCallsBuildXLServer()
{
string fileId = "142342:2";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestAddBuildXLFile_UploadCallsBuildXLServer) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var fileInfo = item.EnsureMaterialized().GetAwaiter().GetResult();
XAssert.IsTrue(fileInfo != null && fileInfo.Exists);
XAssert.AreEqual(TestFileContent, File.ReadAllText(fileInfo.FullName));
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(client, dropName, filePath, fileId, fileContentInfo: TestFileContentInfo);
// addfile succeeds
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsTrue(result.Succeeded, result.Payload);
});
});
}
public async Task TestCommandThatRequestsStopReceivesResponseBeforeDisconnectResponse()
{
const string StopCmd = "stop";
const string StopResponse = "stopped";
IServer theServer = null; // set at the beginning of testAction
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
Assert.Equal(StopCmd, op.Payload);
Assert.NotNull(theServer);
theServer.RequestStop();
return(IpcResult.Success(StopResponse));
});
await WithServerAndClient(
nameof(TestCommandThatRequestsStopReceivesResponseBeforeDisconnectResponse),
ipcExecutor,
maxConcurrentClients : 1,
maxConcurrentRequestsPerClient : 1,
testAction : async(server, tcpProvider, client) =>
{
theServer = server;
// send stop request
var stopRequest = new Request(new IpcOperation(StopCmd, waitForServerAck: true));
await stopRequest.SerializeAsync(client);
// assert response to the stop request is received first
var resp = await Response.DeserializeAsync(client);
Assert.Equal(stopRequest.Id, resp.RequestId);
Assert.True(resp.Result.Succeeded, "Expected response to succeed; error: " + resp.Result.Payload);
Assert.Equal(StopResponse, resp.Result.Payload);
// assert disconnect response is received next
resp = await Response.DeserializeAsync(client);
Assert.True(resp.IsDisconnectResponse, "Expected a disconnect response, got: " + resp);
});
}
public async Task TestMaxConcurrentRequestPerClient()
{
var mre = new ManualResetEvent(false);
var waitReq = new Request(new IpcOperation("wait", waitForServerAck: true));
var signalReq = new Request(new IpcOperation("signal", waitForServerAck: true));
var waitSignalExecutor = new LambdaIpcOperationExecutor(op =>
{
if (op.Payload == waitReq.Operation.Payload)
{
mre.WaitOne();
return(IpcResult.Success());
}
if (op.Payload == signalReq.Operation.Payload)
{
mre.Set();
return(IpcResult.Success());
}
Assert.False(true);
return(null);
});
// with 2 max concurrent requests
await WithServerAndClient(
nameof(TestMaxConcurrentRequestPerClient) + ".2",
waitSignalExecutor,
maxConcurrentRequestsPerClient : 2,
testAction : async(server, tcpProvider, clientStream) =>
{
// send "wait" first, then "signal"
await waitReq.SerializeAsync(clientStream);
await signalReq.SerializeAsync(clientStream);
// both responses arrive
await Response.DeserializeAsync(clientStream);
await Response.DeserializeAsync(clientStream);
});
mre.Reset();
// with 1 max concurrent requests
await WithServerAndClient(
nameof(TestMaxConcurrentRequestPerClient) + ".1",
waitSignalExecutor,
maxConcurrentClients : 2,
maxConcurrentRequestsPerClient : 1,
testAction : async(server, tcpProvider, client) =>
{
// send "wait" first, then "signal"
await waitReq.SerializeAsync(client);
await signalReq.SerializeAsync(client);
// send signal from a new client
using (var tcpClient2 = await tcpProvider.ConnectToServerAsync())
using (var clientStream2 = tcpClient2.GetStream())
{
await signalReq.SerializeAsync(clientStream2);
var resp = await Response.DeserializeAsync(clientStream2);
Assert.True(resp.Result.Succeeded);
}
// finally both client1 tasks complete
var resp1 = await Response.DeserializeAsync(client);
var resp2 = await Response.DeserializeAsync(client);
Assert.True(resp1.Result.Succeeded);
Assert.True(resp2.Result.Succeeded);
});
}
[InlineData(".*d\\.txt$")] // no files
public void TestAddDirectoryToDropWithFilters(string filter)
{
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- a.txt
// |- b.txt
// |- bar <directory>
// |- c.txt
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <(string fileName, string remoteFileName)>
{
(Path.Combine(directoryPath, "a.txt"), $"{remoteDirectoryPath}/a.txt"),
(Path.Combine(directoryPath, "b.txt"), $"{remoteDirectoryPath}/b.txt"),
(Path.Combine(directoryPath, "bar", "c.txt"), $"{remoteDirectoryPath}/bar/c.txt"),
};
var dropPaths = new List <string>();
var expectedDropPaths = new HashSet <string>();
var regex = new Regex(filter, RegexOptions.IgnoreCase | RegexOptions.Compiled);
expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName));
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public void TestAddDirectoryToDrop()
{
var dropPaths = new System.Collections.Generic.List <string>();
var expectedDropPaths = new System.Collections.Generic.HashSet <string>();
string directoryId = "123:1:12345";
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- b.txt
// |- c.txt
// |- bar <directory>
// |- d.txt
var path = Path.Combine(TestOutputDirectory, "foo");
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
Directory.CreateDirectory(path);
var fileFooB = Path.Combine(TestOutputDirectory, "foo", "b.txt");
File.WriteAllText(fileFooB, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/b.txt");
var fileFooC = Path.Combine(TestOutputDirectory, "foo", "c.txt");
File.WriteAllText(fileFooC, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/c.txt");
path = Path.Combine(TestOutputDirectory, "foo", "bar");
Directory.CreateDirectory(path);
var fileFooBarD = Path.Combine(TestOutputDirectory, "foo", "bar", "d.txt");
File.WriteAllText(fileFooBarD, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/bar/d.txt");
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, directoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = new System.Collections.Generic.List <SealedDirectoryFile>
{
CreateFakeSealedDirectoryFile(fileFooB),
CreateFakeSealedDirectoryFile(fileFooC),
CreateFakeSealedDirectoryFile(fileFooBarD)
};
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithSetup(dropClient, async(daemon, etwListener) =>
{
await WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
async(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var ipcResult = await daemon.AddDirectoryAsync(Path.Combine(TestOutputDirectory, "foo"), directoryId, "remote", TestChunkDedup, client);
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
});
});
}
public void TestAdddingDirectoryToDropWithSpecifiedRelativePathReplacement(string dropPath, string replaceOldValue, string replaceNewValue, params string[] expectedFiles)
{
/*
* Directory content:
* a
* dir\b
* dir\dir\c
* dir\dir2\d
* dir3\e
*/
var expectedDropPaths = new HashSet <string>(expectedFiles);
var dropPaths = new List <string>();
var fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <string>
{
Path.Combine(directoryPath, "a"),
Path.Combine(directoryPath, "dir", "b"),
Path.Combine(directoryPath, "dir", "dir", "c"),
Path.Combine(directoryPath, "dir", "dir2", "d"),
Path.Combine(directoryPath, "dir3", "e"),
};
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(file => CreateFakeSealedDirectoryFile(file)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener, dropConfig) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {dropPath} --directoryFilter .* --directoryRelativePathReplace {serializeReplaceArgument(replaceOldValue, replaceNewValue)} --directoryFilterUseRelativePath false",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
string serializeReplaceArgument(string oldValue, string newValue)
{
if (oldValue != null || newValue != null)
{
return($"#{oldValue}#{newValue}#");
}
return("##");
}
}
