public void TestAddDirectoryToDropWithEmptyRelativePath()
{
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <(string fileName, string remoteFileName)>
{
(Path.Combine(directoryPath, "a.txt"), "a.txt"),
(Path.Combine(directoryPath, "b.txt"), "b.txt"),
(Path.Combine(directoryPath, "bar", "c.txt"), "bar/c.txt"),
};
var dropPaths = new List <string>();
var expectedDropPaths = new HashSet <string>();
var regex = new Regex(".*", RegexOptions.IgnoreCase | RegexOptions.Compiled);
expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName));
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath . --directoryFilter .*",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
return(IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig));
}
public void TestAdddingDirectoryToDropWithSpecifiedRelativePathReplacement(string dropPath, string replaceOldValue, string replaceNewValue, params string[] expectedFiles)
{
/*
* Directory content:
* a
* dir\b
* dir\dir\c
* dir\dir2\d
* dir3\e
*/
var expectedDropPaths = new HashSet <string>(expectedFiles);
var dropPaths = new List <string>();
var fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <string>
{
Path.Combine(directoryPath, "a"),
Path.Combine(directoryPath, "dir", "b"),
Path.Combine(directoryPath, "dir", "dir", "c"),
Path.Combine(directoryPath, "dir", "dir2", "d"),
Path.Combine(directoryPath, "dir3", "e"),
};
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(file => CreateFakeSealedDirectoryFile(file)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener, dropConfig) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {dropPath} --directoryFilter .* --directoryRelativePathReplace {serializeReplaceArgument(replaceOldValue, replaceNewValue)} --directoryFilterUseRelativePath false",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
string serializeReplaceArgument(string oldValue, string newValue)
{
if (oldValue != null || newValue != null)
{
return($"#{oldValue}#{newValue}#");
}
return("##");
}
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create();
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile)
{
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
var file = new SealedDirectoryFile(
Path.Combine(directoryPath, "file.txt"),
new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1),
FileContentInfo.CreateWithUnknownLength(WellKnownContentHashes.AbsentFile));
return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> {
file
})));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener, dropConfig) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --service {dropConfig.Service} --name {dropConfig.Name} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .* --directoryRelativePathReplace ## --directoryFilterUseRelativePath false",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
// if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it
XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded);
XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
