public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile)
{
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
var file = new SealedDirectoryFile(
Path.Combine(directoryPath, "file.txt"),
new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1),
FileContentInfo.CreateWithUnknownLength(global::BuildXL.Scheduler.WellKnownContentHashes.AbsentFile));
return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> {
file
})));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
// if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it
XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded);
XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public void TestAddFile_AssociateDoesntNeedServer()
{
// this client only touches item.BlobIdentifier and returns 'Associated'
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
return(Task.FromResult(AddFileResult.Associated));
});
WithSetup(dropClient, (daemon, etwListener) =>
{
var provider = IpcFactory.GetProvider();
var connStr = provider.CreateNewConnectionString();
var contentInfo = new FileContentInfo(new ContentHash(HashType.Vso0), length: 123456);
var client = new Client(provider.GetClient(connStr, new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(client, filePath: "file-which-doesnt-exist.txt", fileId: "23423423:1", chunkDedup: TestChunkDedup, fileContentInfo: contentInfo);
// addfile succeeds without needing BuildXL server nor the file on disk
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsTrue(result.Succeeded);
// calling MaterializeFile fails because no BuildXL server is running
Assert.Throws <DaemonException>(() => addFileItem.EnsureMaterialized().GetAwaiter().GetResult());
});
}
public void TestWithThreads(int numServices, int numRequestsPerService)
{
var ipcProvider = new IpcProviderWithMemoization(IpcFactory.GetProvider());
var ipcMonikers = Enumerable
.Range(0, numServices)
.Select(_ => ipcProvider.RenderConnectionString(ipcProvider.CreateNewMoniker()))
.ToList();
var serverThreads = ipcMonikers
.Select(moniker => CreateThreadForCommand($"{StartNoDropCmd.Name} --{Moniker.LongName} " + moniker, null))
.ToList();
Start(serverThreads);
Thread.Sleep(100);
var clientThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, numRequestsPerService, $"{PingDaemonCmd.Name} --{Moniker.LongName} <moniker>");
Start(clientThreads);
Join(clientThreads);
var serverShutdownThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, 1, $"{StopDaemonCmd.Name} --{Moniker.LongName} <moniker>");
Start(serverShutdownThreads);
Join(serverShutdownThreads);
Join(serverThreads);
}
public void TestAlwaysFailWhenAddingStaticallyListedFilesWithAbsentFileHash(bool isSourceFile)
{
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.CreateNewConnectionString(), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
// only hash and file rewrite count are important here; the rest are just fake values
var hash = FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.CreateSpecialValue(1)).Render();
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {daemon.Config.Moniker} --file non-existent-file.txt --dropPath remote-file-name.txt --hash {hash} --fileId 12345:{(isSourceFile ? 0 : 1)}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
XAssert.IsFalse(ipcResult.Succeeded);
XAssert.AreEqual(IpcResultStatus.InvalidInput, ipcResult.ExitCode);
},
bxlApiClient);
}
public async Task TestWithThreadsAsync(int numServices, int numRequestsPerService)
{
var ipcProvider = new IpcProviderWithMemoization(IpcFactory.GetProvider());
var ipcMonikers = Enumerable
.Range(0, numServices)
.Select(_ => ipcProvider.RenderConnectionString(ipcProvider.CreateNewMoniker()))
.ToList();
var serverTasks = ipcMonikers
.Select(moniker => CreateTaskForCommand($"{StartNoDropCmd.Name} --{Moniker.LongName} " + moniker, null))
.ToList();
// make sure that the daemons are running
await Task.Delay(200);
var clientThreads = GetClientTasks(ipcProvider, ipcMonikers, numRequestsPerService, $"{PingDaemonCmd.Name} --{Moniker.LongName} <moniker>");
await Task.WhenAll(clientThreads);
var serverShutdownThreads = GetClientTasks(ipcProvider, ipcMonikers, 1, $"{StopDaemonCmd.Name} --{Moniker.LongName} <moniker>");
await Task.WhenAll(serverShutdownThreads);
await Task.WhenAll(serverTasks);
}
public void TestClientFactory()
{
var provider = IpcFactory.GetProvider();
var moniker = provider.CreateNewMoniker();
using var apiClient = Client.Create(provider.RenderConnectionString(moniker));
}
/// <summary>
/// Creates an instance of <see cref="PipGraphFragmentBuilder"/>.
/// </summary>
public PipGraphFragmentBuilder(
PipExecutionContext pipExecutionContext,
IConfiguration configuration,
PathExpander pathExpander)
{
Contract.Requires(pipExecutionContext != null);
Configuration = configuration;
m_pipExecutionContext = pipExecutionContext;
m_lazyApiServerMoniker = configuration.Schedule.UseFixedApiServerMoniker
? Lazy.Create(() => IpcFactory.GetFixedMoniker())
: Lazy.Create(() => IpcFactory.GetProvider().CreateNewMoniker());
SealDirectoryTable = new SealedDirectoryTable(m_pipExecutionContext.PathTable);
if (configuration.Schedule.ComputePipStaticFingerprints)
{
var extraFingerprintSalts = new ExtraFingerprintSalts(
configuration,
PipFingerprintingVersion.TwoPhaseV2,
configuration.Cache.CacheSalt,
new DirectoryMembershipFingerprinterRuleSet(configuration, pipExecutionContext.StringTable).ComputeSearchPathToolsHash());
m_pipStaticFingerprinter = new PipStaticFingerprinter(
pipExecutionContext.PathTable,
sealDirectoryFingerprintLookup: null,
directoryProducerFingerprintLookup: null,
extraFingerprintSalts: extraFingerprintSalts,
pathExpander: pathExpander)
{
FingerprintTextEnabled = configuration.Schedule.LogPipStaticFingerprintTexts
};
}
}
/// <summary>
/// Creates an instance of <see cref="GraphFragmentBuilder"/>.
/// </summary>
public GraphFragmentBuilder(LoggingContext loggingContext, PipExecutionContext pipExecutionContext, IConfiguration configuration)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pipExecutionContext != null);
m_loggingContext = loggingContext;
m_pipExecutionContext = pipExecutionContext;
m_lazyApiServerMoniker = configuration.Schedule.UseFixedApiServerMoniker
? Lazy.Create(() => IpcFactory.GetFixedMoniker())
: Lazy.Create(() => IpcFactory.GetProvider().CreateNewMoniker());
SealDirectoryTable = new SealedDirectoryTable(m_pipExecutionContext.PathTable);
}
public void TestLazilyMaterializedSymlinkRejected()
{
string fileId = "142342:3";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestLazilyMaterializedSymlinkRejected) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var ex = Assert.Throws <DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
// rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it)
throw ex;
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this mock BuildXL server materializes a regular file, which we will treat as a symlink in this test
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(
symlinkTester: (file) => file == filePath ? true : false,
client: client,
fullDropName: dropName,
filePath: filePath,
fileId: fileId,
fileContentInfo: TestFileContentInfo);
// addfile files
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsFalse(result.Succeeded, "expected addfile to fail; instead it succeeded and returned payload: " + result.Payload);
XAssert.IsTrue(result.Payload.Contains(DropItemForBuildXLFile.MaterializationResultIsSymlinkErrorPrefix));
});
});
}
/// <nodoc />
public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, IIpcLogger logger, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Config = daemonConfig;
m_parser = parser;
ApiClient = client;
m_logger = logger;
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
}
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxConnectionKext = SandboxConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
TempCleaner = new TestMoveDeleteCleaner(tempDirectory);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
public void TestAddBuildXLFile_UploadCallsBuildXLServer()
{
string fileId = "142342:2";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestAddBuildXLFile_UploadCallsBuildXLServer) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var fileInfo = item.EnsureMaterialized().GetAwaiter().GetResult();
XAssert.IsTrue(fileInfo != null && fileInfo.Exists);
XAssert.AreEqual(TestFileContent, File.ReadAllText(fileInfo.FullName));
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(client, dropName, filePath, fileId, fileContentInfo: TestFileContentInfo);
// addfile succeeds
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsTrue(result.Succeeded, result.Payload);
});
});
}
/// <nodoc />
public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Contract.Requires(dropConfig != null);
Config = daemonConfig;
DropConfig = dropConfig;
m_parser = parser;
ApiClient = client;
m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig));
}
private PluginCreationArgument GetPluginArgument(string pluginPath, bool runInSeparateProcess)
{
var pluginId = PluginFactory.Instance.CreatePluginId();
return(new PluginCreationArgument()
{
PluginPath = pluginPath,
RunInSeparateProcess = runInSeparateProcess,
PluginId = pluginId,
ConnectionOption = new PluginConnectionOption()
{
IpcMoniker = IpcFactory.GetProvider().LoadAndRenderMoniker(IpcFactory.GetProvider().CreateNewMoniker().Id),
LogDir = m_logDirectory,
Logger = PluginLogUtils.CreateLoggerForPluginClients(m_loggingContext, pluginId)
},
CreatePluginClientFunc = options =>
{
int port = TcpIpConnectivity.ParsePortNumber(options.IpcMoniker);
return new PluginClient(IPAddress.Loopback.ToString(), port, options.Logger);
}
});
}
/// <summary>
/// Convenient factory method.
/// </summary>
public static Client Create(string connectionString, IClientConfig config = null)
{
var provider = IpcFactory.GetProvider();
return(new Client(provider.GetClient(connectionString, config ?? new ClientConfig())));
}
/// <summary>
/// Gets an IPC moniker.
/// </summary>
public IIpcMoniker GetIpcMoniker(PipConstructionHelper helper = null)
{
var semiStableHash = (helper ?? m_defaultConstructionHelper).GetNextSemiStableHash();
return(IpcFactory.GetProvider().LoadOrCreateMoniker(string.Format(CultureInfo.InvariantCulture, "{0:X16}", semiStableHash)));
}
public void TestAdddingDirectoryToDropWithSpecifiedRelativePathReplacement(string dropPath, string replaceOldValue, string replaceNewValue, params string[] expectedFiles)
{
/*
* Directory content:
* a
* dir\b
* dir\dir\c
* dir\dir2\d
* dir3\e
*/
var expectedDropPaths = new HashSet <string>(expectedFiles);
var dropPaths = new List <string>();
var fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <string>
{
Path.Combine(directoryPath, "a"),
Path.Combine(directoryPath, "dir", "b"),
Path.Combine(directoryPath, "dir", "dir", "c"),
Path.Combine(directoryPath, "dir", "dir2", "d"),
Path.Combine(directoryPath, "dir3", "e"),
};
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(file => CreateFakeSealedDirectoryFile(file)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener, dropConfig) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {dropPath} --directoryFilter .* --directoryRelativePathReplace {serializeReplaceArgument(replaceOldValue, replaceNewValue)} --directoryFilterUseRelativePath false",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
string serializeReplaceArgument(string oldValue, string newValue)
{
if (oldValue != null || newValue != null)
{
return($"#{oldValue}#{newValue}#");
}
return("##");
}
}
public void TestAddDirectoryToDrop()
{
var dropPaths = new System.Collections.Generic.List <string>();
var expectedDropPaths = new System.Collections.Generic.HashSet <string>();
string directoryId = "123:1:12345";
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- b.txt
// |- c.txt
// |- bar <directory>
// |- d.txt
var path = Path.Combine(TestOutputDirectory, "foo");
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
Directory.CreateDirectory(path);
var fileFooB = Path.Combine(TestOutputDirectory, "foo", "b.txt");
File.WriteAllText(fileFooB, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/b.txt");
var fileFooC = Path.Combine(TestOutputDirectory, "foo", "c.txt");
File.WriteAllText(fileFooC, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/c.txt");
path = Path.Combine(TestOutputDirectory, "foo", "bar");
Directory.CreateDirectory(path);
var fileFooBarD = Path.Combine(TestOutputDirectory, "foo", "bar", "d.txt");
File.WriteAllText(fileFooBarD, Guid.NewGuid().ToString());
expectedDropPaths.Add("remote/bar/d.txt");
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, directoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = new System.Collections.Generic.List <SealedDirectoryFile>
{
CreateFakeSealedDirectoryFile(fileFooB),
CreateFakeSealedDirectoryFile(fileFooC),
CreateFakeSealedDirectoryFile(fileFooBarD)
};
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithSetup(dropClient, async(daemon, etwListener) =>
{
await WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
async(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var ipcResult = await daemon.AddDirectoryAsync(Path.Combine(TestOutputDirectory, "foo"), directoryId, "remote", TestChunkDedup, client);
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
});
});
}
private static EvaluationResult GetNewIpcMoniker(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
var semiStableHash = context.GetPipConstructionHelper().GetNextSemiStableHash();
return(EvaluationResult.Create(IpcFactory.GetProvider().LoadOrCreateMoniker(string.Format(CultureInfo.InvariantCulture, "{0:X16}", semiStableHash))));
}
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
return(IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig));
}
/// <summary>
/// Convenient factory method.
/// </summary>
public static Client Create(string moniker, IClientConfig config = null)
{
var provider = IpcFactory.GetProvider();
return(new Client(provider.GetClient(moniker, config ?? new ClientConfig())));
}
[InlineData(".*d\\.txt$")] // no files
public void TestAddDirectoryToDropWithFilters(string filter)
{
// TestOutputDirectory
// |- foo <directory> <- 'uploading' this directory
// |- a.txt
// |- b.txt
// |- bar <directory>
// |- c.txt
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <(string fileName, string remoteFileName)>
{
(Path.Combine(directoryPath, "a.txt"), $"{remoteDirectoryPath}/a.txt"),
(Path.Combine(directoryPath, "b.txt"), $"{remoteDirectoryPath}/b.txt"),
(Path.Combine(directoryPath, "bar", "c.txt"), $"{remoteDirectoryPath}/bar/c.txt"),
};
var dropPaths = new List <string>();
var expectedDropPaths = new HashSet <string>();
var regex = new Regex(filter, RegexOptions.IgnoreCase | RegexOptions.Compiled);
expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName));
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
