private static async Task <IIpcResult> RegisterManifestInternalAsync(ConfiguredCommand conf, MaterializationDaemon daemon)
{
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}")));
}
if (daemon.ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var manifests = new List <SealedDirectoryFile>();
for (int i = 0; i < directoryIds.Length; i++)
{
var directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryIds[i]);
var possibleContent = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPaths[i]);
if (!possibleContent.Succeeded)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Failed to get the content of a directory artifact ({directoryIds[i]}, {directoryPaths[i]}){Environment.NewLine}{possibleContent.Failure.DescribeIncludingInnerFailures()}")));
}
manifests.AddRange(possibleContent.Result);
}
daemon.Logger.Verbose(string.Join(Environment.NewLine, manifests.Select(f => f.FileName)));
// TODO: placeholder for now - to be implemented in another pr
return(IpcResult.Success("done"));
}
private async Task <IIpcResult> MaterializeDirectoriesAsync(ConfiguredCommand conf)
{
m_counters.IncrementCounter(MaterializationDaemonCounter.MaterializeDirectoriesRequestCount);
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
var directoryFilterKinds = DirectoryContentFilterKind.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryFilters.Length || directoryPaths.Length != directoryFilterKinds.Length)
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #directoryFilters = {directoryFilters.Length}, #directoryFilterKinds = {directoryFilterKinds.Length}")));
}
if (ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var possibleFilterKinds = ParseFilterKinds(directoryFilterKinds);
if (!possibleFilterKinds.Succeeded)
{
return(new IpcResult(IpcResultStatus.InvalidInput, possibleFilterKinds.Failure.Describe()));
}
var initializedFilters = possibleFilters.Result;
var filterKinds = possibleFilterKinds.Result;
var possibleFiles = await GetUniqueFilteredDirectoryContentAsync(directoryPaths, directoryIds, initializedFilters, filterKinds);
if (!possibleFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFiles.Failure.Describe()));
}
var filesToMaterialize = possibleFiles.Result;
var sw = Stopwatch.StartNew();
using (m_counters.StartStopwatch(MaterializationDaemonCounter.MaterializeDirectoriesOuterMaterializationDuration))
{
try
{
await m_actionQueue.ForEachAsync(
filesToMaterialize,
async (file, i) =>
{
m_counters.AddToCounter(MaterializationDaemonCounter.MaterializeDirectoriesMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(file.Artifact, file.FileName, ignoreMaterializationFailures: false);
});
}
catch (Exception e)
{
return(new IpcResult(
IpcResultStatus.GenericError,
e.ToStringDemystified()));
}
}
m_counters.IncrementCounter(MaterializationDaemonCounter.MaterializeDirectoriesFilesToMaterialize);
return(IpcResult.Success(
$"Materialized files ({filesToMaterialize.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, filesToMaterialize.Select(f => f.FileName))}"));
}
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
{
var files = File.GetValues(conf.Config).ToArray();
var fileIds = FileId.GetValues(conf.Config).ToArray();
var hashes = HashOptional.GetValues(conf.Config).ToArray();
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}")));
}
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}")));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var dropFileItemsKeyedByIsAbsent = Enumerable
.Range(0, files.Length)
.Select(i => new DropItemForBuildXLFile(
daemon.ApiClient,
filePath: files[i],
fileId: fileIds[i],
fileContentInfo: FileContentInfo.Parse(hashes[i]),
relativeDropPath: dropPaths[i]))
.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// If a user specified a particular file to be added to drop, this file must be a part of drop.
// The missing files will not get into the drop, so we emit an error.
if (dropFileItemsKeyedByIsAbsent[true].Any())
{
string missingFiles = string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true].Select(f => $"{f.FullFilePath} ({f})"));
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Cannot add the following files to drop because they do not exist:{Environment.NewLine}{missingFiles}")));
}
(IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
conf,
daemon,
directoryPaths,
directoryIds,
directoryDropPaths,
possibleFilters.Result);
if (error != null)
{
return(new IpcResult(IpcResultStatus.ExecutionError, error));
}
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}")));
}
// return early if there is nothing to upload
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
{
return(new IpcResult(IpcResultStatus.Success, string.Empty));
}
return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false])));
}
/// <summary>
/// Takes a list of sealed directories, searches their contents for special manifest files, parses each of them to obtain
/// a list of files to materialize, and finally requests from BuildXL's ApiServer to materialize them.
/// </summary>
private async Task <IIpcResult> RegisterManifestInternalAsync(ConfiguredCommand conf)
{
m_counters.IncrementCounter(MaterializationDaemonCounter.RegisterManifestRequestCount);
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #directoryFilters = {directoryFilters.Length}")));
}
if (ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var initializedFilters = possibleFilters.Result;
var possibleManifestFiles = await GetUniqueFilteredDirectoryContentAsync(directoryPaths, directoryIds, initializedFilters);
if (!possibleManifestFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleManifestFiles.Failure.Describe()));
}
var manifestFiles = possibleManifestFiles.Result;
var sw = Stopwatch.StartNew();
List <string> filesToMaterialize = null;
try
{
// ensure that the manifest files are actually present on disk
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
manifestFiles,
async (manifest, i) =>
{
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(manifest.Artifact, manifest.FileName, ignoreMaterializationFailures: false);
});
}
Possible <List <string> > possibleFiles;
using (m_counters.StartStopwatch(MaterializationDaemonCounter.ManifestParsingOuterDuration))
{
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingTotalFiles, possibleManifestFiles.Result.Count);
possibleFiles = await ParseManifestFilesAsync(manifestFiles);
if (!possibleFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFiles.Failure.Describe()));
}
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingReferencedTotalFiles, possibleFiles.Result.Count);
}
filesToMaterialize = possibleFiles.Result;
sw = Stopwatch.StartNew();
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
filesToMaterialize,
async (file, i) =>
{
// Since these file paths are not real build artifacts (i.e., just strings constructed by a parser),
// they might not be "known" to BuildXL, and because of that, BuildXL won't be able to materialize them.
// Manifests are known to contain references to files that are not produced during a build, so we are
// ignoring materialization failures for such files. We are doing this so we won't fail IPC pips and
// a build could succeed.
// If there are real materialization errors (e.g., cache failure), the daemon relies on BuildXL logging
// the appropriate error events and failing the build.
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(FileArtifact.Invalid, file, ignoreMaterializationFailures: true);
});
}
}
catch (Exception e)
{
return(new IpcResult(
IpcResultStatus.GenericError,
e.ToStringDemystified()));
}
// Note: we are not claiming here that all the files in filesToMaterialize were materialized cause we are ignoring materialization failures.
// The real materialization status will be logged during execution of the Finalize command.
return(IpcResult.Success(
$"Processed paths ({filesToMaterialize.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, filesToMaterialize)}"));
}