internal async Task <IIpcResult> AddFileAsync(IDropItem dropItem, Func <string, bool> symlinkTester)
{
Contract.Requires(dropItem != null);
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
if (System.IO.File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
{
return(new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath));
}
return(await WrapDropErrorsIntoIpcResult(async() =>
{
IDropClient dropClient = await m_dropClientTask;
AddFileResult result = await dropClient.AddFileAsync(dropItem);
switch (result)
{
case AddFileResult.Associated:
case AddFileResult.UploadedAndAssociated:
case AddFileResult.SkippedAsDuplicate:
return IpcResult.Success(I($"File '{dropItem.FullFilePath}' {result} under '{dropItem.RelativeDropPath}' in drop '{DropName}'."));
case AddFileResult.RegisterFileForBuildManifestFailure:
return new IpcResult(IpcResultStatus.ExecutionError, $"Failure during BuildManifest Hash generation for File '{dropItem.FullFilePath}' {result} under '{dropItem.RelativeDropPath}' in drop '{DropName}'.");
default:
return new IpcResult(IpcResultStatus.ExecutionError, $"Unhandled drop result: {result}");
}
}));
}
public async Task TestGetSealedDirectoryContentAsync(uint partialSealId, bool isSharedOpaque, int numSealedDirectoryFiles)
{
// skip the invalid configuration
if (partialSealId == 0 && isSharedOpaque)
{
return;
}
var dirPath = X("/a/b/");
var dirArtifact = new DirectoryArtifact(AbsPath(dirPath), partialSealId, isSharedOpaque);
var cmdResult = Enumerable
.Range(0, numSealedDirectoryFiles)
.Select(i => new SealedDirectoryFile($"f{i}", SourceFile(X($"{dirPath}/f{i}")), RandomFileContentInfo()))
.ToList();
using var apiClient = CreateApiClient(ipcOperation =>
{
var cmd = (GetSealedDirectoryContentCommand)Command.Deserialize(ipcOperation.Payload);
XAssert.AreEqual(dirArtifact, cmd.Directory);
XAssert.AreEqual(dirPath, cmd.FullDirectoryPath);
return(IpcResult.Success(cmd.RenderResult(cmdResult)));
});
var maybeResult = await apiClient.GetSealedDirectoryContent(dirArtifact, dirPath);
XAssert.PossiblySucceeded(maybeResult);
XAssert.ArrayEqual(cmdResult.ToArray(), maybeResult.Result.ToArray());
}
private async Task <IIpcResult> ExecuteGetSealedDirectoryContentAsync(GetSealedDirectoryContentCommand cmd)
{
Contract.Requires(cmd != null);
// for extra safety, check that provided directory path and directory id match
AbsolutePath dirPath;
bool isValidPath = AbsolutePath.TryCreate(m_context.PathTable, cmd.FullDirectoryPath, out dirPath);
if (!isValidPath || !cmd.Directory.Path.Equals(dirPath))
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
"directory path ids differ, or could not create AbsolutePath; directory = " + cmd.Directory.Path.ToString(m_context.PathTable) + ", directory path = " + cmd.FullDirectoryPath));
}
var files = m_fileContentManager.ListSealedDirectoryContents(cmd.Directory);
Tracing.Logger.Log.ApiServerGetSealedDirectoryContentExecuted(m_loggingContext, cmd.Directory.Path.ToString(m_context.PathTable), files.Length);
var inputContentsTasks = files
.Select(f => m_fileContentManager.TryQuerySealedOrUndeclaredInputContentAsync(f.Path, nameof(ApiServer), allowUndeclaredSourceReads: true))
.ToArray();
var inputContents = await TaskUtilities.SafeWhenAll(inputContentsTasks);
var results = new List <BuildXL.Ipc.ExternalApi.SealedDirectoryFile>();
var failedResults = new List <string>();
for (int i = 0; i < files.Length; ++i)
{
// If the content has no value or has unknown length, then we have some inconsistency wrt the sealed directory content
// Absent files are an exception since it is possible to have sealed directories with absent files (shared opaques is an example of this).
// In those cases we leave the consumer to deal with them.
if (!inputContents[i].HasValue || (inputContents[i].Value.Hash != WellKnownContentHashes.AbsentFile && !inputContents[i].Value.HasKnownLength))
{
failedResults.Add(files[i].Path.ToString(m_context.PathTable));
}
else
{
results.Add(new BuildXL.Ipc.ExternalApi.SealedDirectoryFile(
files[i].Path.ToString(m_context.PathTable),
files[i],
inputContents[i].Value));
}
}
if (failedResults.Count > 0)
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
string.Format("Could not find content information for {0} out of {1} files inside of '{4}':{2}{3}",
failedResults.Count,
files.Length,
Environment.NewLine,
string.Join("; ", failedResults),
cmd.Directory.Path.ToString(m_context.PathTable))));
}
return(IpcResult.Success(cmd.RenderResult(results)));
}
/// <summary>
/// Executes <see cref="RegisterFilesForBuildManifestCommand"/>. Checks if Cache contains build manifest hashes for given <see cref="BuildManifestEntry"/>.
/// Else checks if local files exist and computes their ContentHash.
/// Returns an empty array on success. Any failing BuildManifestEntries are returned for logging.
/// If build manifest hashes are not available, the files are materialized using <see cref="ExecuteMaterializeFileAsync"/>, the build manifest hashes are computed and stored into cache.
/// </summary>
private async Task <IIpcResult> ExecuteRecordBuildManifestHashesAsync(RegisterFilesForBuildManifestCommand cmd)
{
Contract.Requires(cmd != null);
var tasks = cmd.BuildManifestEntries
.Select(buildManifestEntry => ExecuteRecordBuildManifestHashWithXlgAsync(cmd.DropName, buildManifestEntry))
.ToArray();
var result = await TaskUtilities.SafeWhenAll(tasks);
if (result.Any(value => !value.IsValid))
{
BuildManifestEntry[] failures = result.Where(value => !value.IsValid)
.Select(value => new BuildManifestEntry(value.RelativePath, value.AzureArtifactsHash, "Invalid", FileArtifact.Invalid)) // FullFilePath is unused by the caller
.ToArray();
return(IpcResult.Success(cmd.RenderResult(failures)));
}
else
{
m_executionLog.RecordFileForBuildManifest(new Tracing.RecordFileForBuildManifestEventData(result.ToList()));
return(IpcResult.Success(cmd.RenderResult(Array.Empty <BuildManifestEntry>())));
}
}
public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile)
{
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
var file = new SealedDirectoryFile(
Path.Combine(directoryPath, "file.txt"),
new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1),
FileContentInfo.CreateWithUnknownLength(global::BuildXL.Scheduler.WellKnownContentHashes.AbsentFile));
return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> {
file
})));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
// if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it
XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded);
XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
/// <summary>
/// Executes <see cref="RegisterFileForBuildManifestCommand"/>. Checks if local file exists and computes it's ContentHash.
/// Else checks if Cache contains SHA-256 Hash for given <see cref="RegisterFileForBuildManifestCommand.Hash"/>.
/// Returns true if SHA-256 ContentHash exists.
/// Else the file is materialized using <see cref="ExecuteMaterializeFileAsync"/>, the build manifest hash is computed and stored into cache.
/// </summary>
private async Task <IIpcResult> ExecuteGetBuildManifestHashAsync(RegisterFileForBuildManifestCommand cmd)
{
Contract.Requires(cmd != null);
if (m_inMemoryBuildManifestStore.TryGetValue(cmd.Hash, out var buildManifestHash))
{
RecordFileForBuildManifestInXLG(cmd.DropName, cmd.RelativePath, cmd.Hash, buildManifestHash);
return(IpcResult.Success(cmd.RenderResult(true)));
}
ContentHash?sha256Hash =
await TryGetBuildManifestHashFromLocalFileAsync(cmd.FullFilePath) ??
await TryGetBuildManifestHashAsync(cmd.Hash);
if (sha256Hash.HasValue)
{
m_inMemoryBuildManifestStore.TryAdd(cmd.Hash, sha256Hash.Value);
RecordFileForBuildManifestInXLG(cmd.DropName, cmd.RelativePath, cmd.Hash, sha256Hash.Value);
return(IpcResult.Success(cmd.RenderResult(true)));
}
var computeHashResult = await ComputeBuildManifestHashFromCacheAsync(cmd);
if (computeHashResult.Succeeded)
{
m_inMemoryBuildManifestStore.TryAdd(cmd.Hash, computeHashResult.Result);
RecordFileForBuildManifestInXLG(cmd.DropName, cmd.RelativePath, cmd.Hash, computeHashResult.Result);
await StoreBuildManifestHashAsync(cmd.Hash, computeHashResult.Result);
return(IpcResult.Success(cmd.RenderResult(true)));
}
Tracing.Logger.Log.ErrorApiServerGetBuildManifestHashFromCacheFailed(m_loggingContext, cmd.Hash.Serialize(), computeHashResult.Failure.DescribeIncludingInnerFailures());
return(IpcResult.Success(cmd.RenderResult(false)));
}
/// <summary>
/// Executes <see cref="MaterializeFileCommand"/>. First check that <see cref="MaterializeFileCommand.File"/>
/// and <see cref="MaterializeFileCommand.FullFilePath"/> match, then delegates to <see cref="FileContentManager.TryMaterializeFileAsync"/>.
/// </summary>
private async Task <IIpcResult> ExecuteMaterializeFileAsync(MaterializeFileCommand cmd)
{
Contract.Requires(cmd != null);
// for extra safety, check that provided file path and file id match
AbsolutePath filePath;
bool isValidPath = AbsolutePath.TryCreate(m_context.PathTable, cmd.FullFilePath, out filePath);
if (!isValidPath || !cmd.File.Path.Equals(filePath))
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
"file path ids differ; file = " + cmd.File.Path.ToString(m_context.PathTable) + ", file path = " + cmd.FullFilePath));
}
var result = await m_fileContentManager.TryMaterializeFileAsync(cmd.File);
bool succeeded = result == ArtifactMaterializationResult.Succeeded;
string absoluteFilePath = cmd.File.Path.ToString(m_context.PathTable);
// if file materialization failed, log an error here immediately, so that this errors gets picked up as the root cause
// (i.e., the "ErrorBucket") instead of whatever fallout ends up happening (e.g., IPC pip fails)
if (!succeeded)
{
Tracing.Logger.Log.ErrorApiServerMaterializeFileFailed(m_loggingContext, absoluteFilePath, result.ToString());
}
else
{
Tracing.Logger.Log.ApiServerMaterializeFileSucceeded(m_loggingContext, absoluteFilePath);
}
return(IpcResult.Success(cmd.RenderResult(succeeded)));
}
/// <summary>
/// Executes <see cref="ReportStatisticsCommand"/>.
/// </summary>
private Task <IIpcResult> ExecuteReportStatistics(ReportStatisticsCommand cmd)
{
Contract.Requires(cmd != null);
Tracing.Logger.Log.ApiServerReportStatisticsExecuted(m_loggingContext, cmd.Stats.Count);
Logger.Log.BulkStatistic(m_loggingContext, cmd.Stats);
return(Task.FromResult(IpcResult.Success(cmd.RenderResult(true))));
}
private Task <IIpcResult> ExecuteReportServicePipIsReadyAsync(ReportServicePipIsReadyCommand cmd)
{
Contract.Requires(cmd != null);
m_serviceManger.ReportServiceIsReady(cmd.ProcessId, cmd.ProcessName);
return(Task.FromResult(IpcResult.Success(cmd.RenderResult(true))));
}
private async Task <IIpcResult> ExecuteGetSealedDirectoryContent(GetSealedDirectoryContentCommand cmd)
{
Contract.Requires(cmd != null);
// for extra safety, check that provided directory path and directory id match
AbsolutePath dirPath;
bool isValidPath = AbsolutePath.TryCreate(m_context.PathTable, cmd.FullDirectoryPath, out dirPath);
if (!isValidPath || !cmd.Directory.Path.Equals(dirPath))
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
"directory path ids differ, or could not create AbsolutePath; directory = " + cmd.Directory.Path.ToString(m_context.PathTable) + ", directory path = " + cmd.FullDirectoryPath));
}
var files = m_fileContentManager.ListSealedDirectoryContents(cmd.Directory);
Tracing.Logger.Log.ApiServerGetSealedDirectoryContentExecuted(m_loggingContext, cmd.Directory.Path.ToString(m_context.PathTable), files.Length);
var inputContentsTasks = files
.Select(f => m_fileContentManager.TryQuerySealedOrUndeclaredInputContentAsync(f.Path, nameof(ApiServer), false))
.ToArray();
var inputContents = await TaskUtilities.SafeWhenAll(inputContentsTasks);
var results = new List <BuildXL.Ipc.ExternalApi.SealedDirectoryFile>();
var failedResults = new List <string>();
for (int i = 0; i < files.Length; ++i)
{
if (!inputContents[i].HasValue || !inputContents[i].Value.HasKnownLength)
{
failedResults.Add(files[i].Path.ToString(m_context.PathTable));
}
else
{
results.Add(new BuildXL.Ipc.ExternalApi.SealedDirectoryFile(
files[i].Path.ToString(m_context.PathTable),
files[i],
inputContents[i].Value));
}
}
if (failedResults.Count > 0)
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
string.Format("Could not find content information for {0} out of {1} files inside of '{4}':{2}{3}",
failedResults.Count,
files.Length,
Environment.NewLine,
string.Join("; ", failedResults),
cmd.Directory.Path.ToString(m_context.PathTable))));
}
return(IpcResult.Success(cmd.RenderResult(results)));
}
/// <summary>
/// Executes <see cref="GenerateBuildManifestDataCommand"/>. Generates a BuildManifest.json file for given
/// <see cref="GenerateBuildManifestDataCommand.DropName"/>.
/// </summary>
private IIpcResult ExecuteGenerateBuildManifestData(GenerateBuildManifestDataCommand cmd)
{
Contract.Requires(cmd != null);
Contract.Requires(m_buildManifestGenerator != null, "Build Manifest data can only be generated on master");
BuildManifestData buildManifestData = m_buildManifestGenerator.GenerateBuildManifestData(cmd.DropName);
return(IpcResult.Success(cmd.RenderResult(buildManifestData)));
}
/// <summary>
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task <IIpcResult> CreateAsync()
{
DropCreationEvent dropCreationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
return(dropCreationEvent.Succeeded
? IpcResult.Success(Inv("Drop {0} created.", DropName))
: new IpcResult(IpcResultStatus.ExecutionError, dropCreationEvent.ErrorMessage));
}
/// <summary>
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task <IIpcResult> FinalizeAsync()
{
var dropFinalizationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
return(dropFinalizationEvent.Succeeded
? IpcResult.Success(Inv("Drop {0} finalized", DropName))
: new IpcResult(IpcResultStatus.ExecutionError, dropFinalizationEvent.ErrorMessage));
}
/// <summary>
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task <IIpcResult> CreateAsync()
{
DropCreationEvent dropCreationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
return(dropCreationEvent.Succeeded
? IpcResult.Success(I($"Drop {DropName} created."))
: new IpcResult(ParseIpcStatus(dropCreationEvent.AdditionalInformation), dropCreationEvent.ErrorMessage));
}
/// <summary>
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task <IIpcResult> FinalizeAsync()
{
var dropFinalizationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
return(dropFinalizationEvent.Succeeded
? IpcResult.Success(I($"Drop {DropName} finalized"))
: new IpcResult(ParseIpcStatus(dropFinalizationEvent.AdditionalInformation), dropFinalizationEvent.ErrorMessage));
}
public async Task TestIntertwinedTwoOperations()
{
await WithSetupAndTeardownAssertingClientCompleted(
nameof(TestIntertwinedTwoOperations),
async (client, serverStream) =>
{
// create 2 operations
var hiOp = new IpcOperation("hi", waitForServerAck: true);
var helloOp = new IpcOperation("hello", waitForServerAck: true);
// send both operations via the same client
var hiTask = Task.Run(() => client.Send(hiOp));
var helloTask = Task.Run(() => client.Send(helloOp));
// receive 2 requests
var req1 = await Request.DeserializeAsync(serverStream);
var req2 = await Request.DeserializeAsync(serverStream);
// aux functions
var matchFn = new Func <IIpcOperation, Request>((op) =>
req1.Operation.Payload == op.Payload ? req1 :
req2.Operation.Payload == op.Payload ? req2 :
null);
var respondFn = new Func <Request, Task>((req) =>
{
var resp = new Response(req.Id, IpcResult.Success(req.Operation.Payload.ToUpperInvariant()));
return(resp.SerializeAsync(serverStream));
});
var waitAndCheckResultFn = new Func <Task <IIpcResult>, IIpcOperation, Task>(async(task, operation) =>
{
var result = await task;
Assert.Equal(operation.Payload.ToUpperInvariant(), result.Payload);
Assert.True(result.Succeeded);
});
// match received requests to sent operations
var hiReq = matchFn(hiOp);
Assert.NotNull(hiReq);
Assert.Equal(hiOp.Payload, hiReq.Operation.Payload);
var helloReq = matchFn(helloOp);
Assert.NotNull(helloReq);
Assert.Equal(helloOp.Payload, helloReq.Operation.Payload);
// respond to helloReq, assert helloTask completes
await respondFn(helloReq);
await waitAndCheckResultFn(helloTask, helloOp);
// respond to hiReq, assert hiTask completes
await respondFn(hiReq);
await waitAndCheckResultFn(hiTask, hiOp);
});
}
/// <summary>
/// Executes <see cref="GenerateBuildManifestFileListCommand"/>. Generates a list of file hashes required for BuildManifest.json file
/// for given <see cref="GenerateBuildManifestFileListCommand.DropName"/>.
/// </summary>
private IIpcResult ExecuteGenerateBuildManifestFileList(GenerateBuildManifestFileListCommand cmd)
{
Contract.Requires(cmd != null);
Contract.Requires(m_buildManifestGenerator != null, "Build Manifest data can only be generated on orchestrator");
if (!m_buildManifestGenerator.TryGenerateBuildManifestFileList(cmd.DropName, out string error, out var buildManifestFileList))
{
return(new IpcResult(IpcResultStatus.ExecutionError, error));
}
return(IpcResult.Success(cmd.RenderResult(buildManifestFileList)));
}
public void TestLazilyMaterializedSymlinkRejected()
{
string fileId = "142342:3";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestLazilyMaterializedSymlinkRejected) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var ex = Assert.Throws <DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
// rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it)
throw ex;
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
// this mock BuildXL server materializes a regular file, which we will treat as a symlink in this test
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(
symlinkTester: (file) => file == filePath ? true : false,
client: client,
fullDropName: dropName,
filePath: filePath,
fileId: fileId,
fileContentInfo: TestFileContentInfo);
// addfile files
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsFalse(result.Succeeded, "expected addfile to fail; instead it succeeded and returned payload: " + result.Payload);
XAssert.IsTrue(result.Payload.Contains(DropItemForBuildXLFile.MaterializationResultIsSymlinkErrorPrefix));
});
});
}
/// <summary>
/// Executes <see cref="RegisterFilesForBuildManifestCommand"/>. Checks if local files exist and computes their ContentHash.
/// Else checks if Cache contains SHA-256 Hashes for given <see cref="BuildManifestEntry"/>.
/// Returns an empty array on success. Any failing BuildManifestEntries are returned for logging.
/// If SHA-256 ContentHashes do not exists, the files are materialized using <see cref="ExecuteMaterializeFileAsync"/>, the build manifest hashes are computed and stored into cache.
/// </summary>
private async Task <IIpcResult> ExecuteRecordBuildManifestHashesAsync(RegisterFilesForBuildManifestCommand cmd)
{
Contract.Requires(cmd != null);
var tasks = cmd.BuildManifestEntries
.Select(buildManifestEntry => ExecuteRecordBuildManifestHashAsync(cmd.DropName, buildManifestEntry))
.ToArray();
var result = await TaskUtilities.SafeWhenAll(tasks);
BuildManifestEntry[] failedEntries = result.Where(value => value != null).ToArray();
return(IpcResult.Success(cmd.RenderResult(failedEntries)));
}
/// <summary>
/// Executes <see cref="MaterializeFileCommand"/>. First check that <see cref="MaterializeFileCommand.File"/>
/// and <see cref="MaterializeFileCommand.FullFilePath"/> match, then delegates to <see cref="FileContentManager.TryMaterializeFileAsync(FileArtifact)"/>.
/// If provided <see cref="MaterializeFileCommand.File"/> is not valid, no checks are done, and the call is delegated
/// to <see cref="FileContentManager.TryMaterializeSealedFileAsync(AbsolutePath)"/>
/// </summary>
private async Task <IIpcResult> ExecuteMaterializeFileAsync(MaterializeFileCommand cmd)
{
Contract.Requires(cmd != null);
// If the FileArtifact was provided, for extra safety, check that provided file path and file id match
AbsolutePath filePath;
bool isValidPath = AbsolutePath.TryCreate(m_context.PathTable, cmd.FullFilePath, out filePath);
if (cmd.File.IsValid && (!isValidPath || !cmd.File.Path.Equals(filePath)))
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
"file path ids differ; file = " + cmd.File.Path.ToString(m_context.PathTable) + ", file path = " + cmd.FullFilePath));
}
// If only path was provided, check that it's a valid path.
else if (!cmd.File.IsValid && !filePath.IsValid)
{
return(new IpcResult(
IpcResultStatus.ExecutionError,
$"failed to create AbsolutePath from '{cmd.FullFilePath}'"));
}
var result = cmd.File.IsValid
? await m_fileContentManager.TryMaterializeFileAsync(cmd.File)
// If file artifact is unknown, try materializing using only the file path.
// This method has lower chance of success, since it depends on FileContentManager's
// ability to infer FileArtifact associated with this path.
: await m_fileContentManager.TryMaterializeSealedFileAsync(filePath);
bool succeeded = result == ArtifactMaterializationResult.Succeeded;
string absoluteFilePath = cmd.File.Path.ToString(m_context.PathTable);
// if file materialization failed, log an error here immediately, so that this errors gets picked up as the root cause
// (i.e., the "ErrorBucket") instead of whatever fallout ends up happening (e.g., IPC pip fails)
if (!succeeded)
{
// For sealed files, materialization might not have succeeded because a path is not known to BXL.
// In such a case, do not log an error, and let the caller deal with the failure.
if (cmd.File.IsValid)
{
Tracing.Logger.Log.ErrorApiServerMaterializeFileFailed(m_loggingContext, absoluteFilePath, cmd.File.IsValid, result.ToString());
}
}
else
{
Tracing.Logger.Log.ApiServerMaterializeFileSucceeded(m_loggingContext, absoluteFilePath);
}
return(IpcResult.Success(cmd.RenderResult(succeeded)));
}
public async Task TestLogMessageAsync(string message, bool isWarning, bool expectedResult)
{
using var apiClient = CreateApiClient(ipcOperation =>
{
var cmd = (LogMessageCommand)Command.Deserialize(ipcOperation.Payload);
XAssert.AreEqual(message, cmd.Message);
XAssert.AreEqual(isWarning, cmd.IsWarning);
return(IpcResult.Success(cmd.RenderResult(expectedResult)));
});
var maybeResult = await apiClient.LogMessage(message, isWarning);
XAssert.PossiblySucceeded(maybeResult);
XAssert.AreEqual(expectedResult, maybeResult.Result);
}
/// <remarks>
/// If an apiClient is not passed (ie. null by default), we creat a new Client that returns success for any bool command called.
/// </remarks>
private void WithSetup(IDropClient dropClient, Action <global::Tool.DropDaemon.DropDaemon, DropEtwListener> action, Client apiClient = null)
{
var etwListener = ConfigureEtwLogging();
string moniker = ServicePipDaemon.IpcProvider.RenderConnectionString(ServicePipDaemon.IpcProvider.CreateNewMoniker());
var daemonConfig = new DaemonConfig(VoidLogger.Instance, moniker: moniker, enableCloudBuildIntegration: false);
var dropConfig = new DropConfig(string.Empty, null);
if (apiClient == null)
{
apiClient = new Client(new MockClient(ipcOperation => IpcResult.Success("true")));
}
var daemon = new global::Tool.DropDaemon.DropDaemon(UnixParser.Instance, daemonConfig, dropConfig, Task.FromResult(dropClient), client: apiClient);
action(daemon, etwListener);
}
/// <summary>
/// Executes <see cref="LogMessageCommand"/>.
/// </summary>
private Task <IIpcResult> ExecuteLogMessage(LogMessageCommand cmd)
{
Contract.Requires(cmd != null);
if (cmd.IsWarning)
{
Tracing.Logger.Log.ApiServerReceivedWarningMessage(m_loggingContext, cmd.Message);
}
else
{
Tracing.Logger.Log.ApiServerReceivedMessage(m_loggingContext, cmd.Message);
}
return(Task.FromResult(IpcResult.Success(cmd.RenderResult(true))));
}
public async Task TestGenerateBuildManifestDataAsync()
{
string dropName = "DropName";
BuildManifestData expectedData = new BuildManifestData("Version", 1598291222, "cbId", "Repo", "branch", "commitId", new List <BuildManifestFile>());
using var apiClient = CreateApiClient(ipcOperation =>
{
var cmd = (GenerateBuildManifestDataCommand)Command.Deserialize(ipcOperation.Payload);
XAssert.AreEqual(dropName, cmd.DropName);
return(IpcResult.Success(cmd.RenderResult(expectedData)));
});
var maybeResult = await apiClient.GenerateBuildManifestData(dropName, "Repo", "branch", "commitId", "cbId");
XAssert.PossiblySucceeded(maybeResult);
XAssert.AreEqual(expectedData, maybeResult.Result);
}
private async Task SendRequestAsync(Request request)
{
request.Operation.Timestamp.Request_BeforeSendTime = DateTime.UtcNow;
Logger.Verbose($"Sending request #{request.Id}");
await request.SerializeAsync(m_stream);
Logger.Verbose($"Sent request #{request.Id}");
request.Operation.Timestamp.Request_AfterSendTime = DateTime.UtcNow;
if (!request.Operation.ShouldWaitForServerAck)
{
await SetResponseAsync(new Response(request.Id, IpcResult.Success()));
}
request.Operation.Timestamp.Request_AfterServerAckTime = DateTime.UtcNow;
}
public async Task TestReportStatisticsAsync(int numStats, bool expectedResult)
{
var stats = Enumerable
.Range(1, numStats)
.ToDictionary(i => $"key{i}", i => (long)new Random().Next());
using var apiClient = CreateApiClient(ipcOperation =>
{
var cmd = (ReportStatisticsCommand)Command.Deserialize(ipcOperation.Payload);
XAssert.SetEqual(stats, cmd.Stats);
return(IpcResult.Success(cmd.RenderResult(expectedResult)));
});
var maybeResult = await apiClient.ReportStatistics(stats);
XAssert.PossiblySucceeded(maybeResult);
XAssert.AreEqual(expectedResult, maybeResult.Result);
}
/// <summary>
/// Executes <see cref="ReportStatisticsCommand"/>.
/// </summary>
private Task <IIpcResult> ExecuteReportStatistics(ReportStatisticsCommand cmd)
{
Contract.Requires(cmd != null);
Tracing.Logger.Log.ApiServerReportStatisticsExecuted(m_loggingContext, cmd.Stats.Count);
foreach (var statistic in cmd.Stats)
{
// we aggregate the stats based on their name
m_receivedStatistics.AddOrUpdate(
statistic.Key,
statistic.Value,
static (key, value) => value,
static (key, newValue, oldValue) => newValue + oldValue);
}
return(Task.FromResult(IpcResult.Success(cmd.RenderResult(true))));
}
internal async Task <IIpcResult> AddFileAsync(IDropItem dropItem, Func <string, bool> symlinkTester)
{
Contract.Requires(dropItem != null);
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
if (System.IO.File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
{
return(new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath));
}
return(await WrapDropErrorsIntoIpcResult(async() =>
{
IDropClient dropClient = await m_dropClientTask;
AddFileResult result = await dropClient.AddFileAsync(dropItem);
return IpcResult.Success(I($"File '{dropItem.FullFilePath}' {result} under '{dropItem.RelativeDropPath}' in drop '{DropName}'."));
}));
}
public async Task TestMaterializeFileAsync(bool isSourceFile, bool expectedResult)
{
var path = X("/x/y/z");
var fileArtifact = isSourceFile ? SourceFile(path) : OutputFile(path);
using var apiClient = CreateApiClient(ipcOperation =>
{
var cmd = (MaterializeFileCommand)Command.Deserialize(ipcOperation.Payload);
XAssert.AreEqual(fileArtifact, cmd.File);
XAssert.AreEqual(path, cmd.FullFilePath);
return(IpcResult.Success(cmd.RenderResult(expectedResult)));
});
var maybeResult = await apiClient.MaterializeFile(fileArtifact, path);
XAssert.PossiblySucceeded(maybeResult);
XAssert.AreEqual(expectedResult, maybeResult.Result);
}
public void TestAddBuildXLFile_UploadCallsBuildXLServer()
{
string fileId = "142342:2";
string filePath = Path.Combine(TestOutputDirectory, nameof(TestAddBuildXLFile_UploadCallsBuildXLServer) + "-test.txt");
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// this client wants to read the file
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var fileInfo = item.EnsureMaterialized().GetAwaiter().GetResult();
XAssert.IsTrue(fileInfo != null && fileInfo.Exists);
XAssert.AreEqual(TestFileContent, File.ReadAllText(fileInfo.FullName));
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
WithSetup(dropClient, (daemon, etwListener, dropConfig) =>
{
var dropName = GetDropFullName(dropConfig);
var ipcProvider = IpcFactory.GetProvider();
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveMaterializeFileCmdAndCheckItMatchesFileId(op.Payload, fileId);
File.WriteAllText(filePath, TestFileContent);
return(IpcResult.Success(cmd.RenderResult(true)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var client = new Client(ipcProvider.GetClient(ipcProvider.RenderConnectionString(moniker), new ClientConfig()));
var addFileItem = new DropItemForBuildXLFile(client, dropName, filePath, fileId, fileContentInfo: TestFileContentInfo);
// addfile succeeds
IIpcResult result = daemon.AddFileAsync(addFileItem).GetAwaiter().GetResult();
XAssert.IsTrue(result.Succeeded, result.Payload);
});
});
}