private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon) { var files = File.GetValues(conf.Config).ToArray(); var fileIds = FileId.GetValues(conf.Config).ToArray(); var hashes = HashOptional.GetValues(conf.Config).ToArray(); var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray(); if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"))); } var directoryPaths = Directory.GetValues(conf.Config).ToArray(); var directoryIds = DirectoryId.GetValues(conf.Config).ToArray(); var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray(); var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray(); if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"))); } (Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters); if (filterInitError != null) { return(new IpcResult(IpcResultStatus.ExecutionError, filterInitError)); } var dropFileItemsKeyedByIsAbsent = Enumerable .Range(0, files.Length) .Select(i => new DropItemForBuildXLFile( daemon.ApiClient, chunkDedup: conf.Get(EnableChunkDedup), filePath: files[i], fileId: fileIds[i], fileContentInfo: FileContentInfo.Parse(hashes[i]), relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // If a user specified a particular file to be added to drop, this file must be a part of drop. // The missing files will not get into the drop, so we emit an error. if (dropFileItemsKeyedByIsAbsent[true].Any()) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"The following files are missing, but they are a part of the drop command:{Environment.NewLine}{string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])}"))); } (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync( conf, daemon, directoryPaths, directoryIds, directoryDropPaths, initializedFilters); if (error != null) { return(new IpcResult(IpcResultStatus.ExecutionError, error)); } var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files) if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile)) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"))); } // return early if there is nothing to upload if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any()) { return(new IpcResult(IpcResultStatus.Success, string.Empty)); } return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]))); }
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon) { var files = File.GetValues(conf.Config).ToArray(); var fileIds = FileId.GetValues(conf.Config).ToArray(); var hashes = HashOptional.GetValues(conf.Config).ToArray(); var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray(); if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"))); } var directoryPaths = Directory.GetValues(conf.Config).ToArray(); var directoryIds = DirectoryId.GetValues(conf.Config).ToArray(); var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray(); var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray(); if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"))); } var possibleFilters = InitializeFilters(directoryFilters); if (!possibleFilters.Succeeded) { return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe())); } ContentHash[] parsedHashes; try { parsedHashes = hashes.Select(hash => FileContentInfo.Parse(hash).Hash).ToArray(); } catch (ArgumentException e) { return(new IpcResult(IpcResultStatus.InvalidInput, "Content Hash Parsing exception: " + e.InnerException)); } if (daemon.DropConfig.EnableBuildManifestCreation) { var buildManifestHashTasks = Enumerable .Range(0, parsedHashes.Length) .Select(i => RegisterFileForBuildManifestAsync(daemon, dropPaths[i], parsedHashes[i], BuildXL.Ipc.ExternalApi.FileId.Parse(fileIds[i]), files[i])); var buildManifestHashes = await TaskUtilities.SafeWhenAll(buildManifestHashTasks); if (buildManifestHashes.Any(h => h == false)) { return(new IpcResult(IpcResultStatus.ExecutionError, "Failure during BuildManifest Hash generation")); } } var dropFileItemsKeyedByIsAbsent = Enumerable .Range(0, files.Length) .Select(i => new DropItemForBuildXLFile( daemon.ApiClient, filePath: files[i], fileId: fileIds[i], fileContentInfo: FileContentInfo.Parse(hashes[i]), relativeDropPath: dropPaths[i])) .ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // If a user specified a particular file to be added to drop, this file must be a part of drop. // The missing files will not get into the drop, so we emit an error. if (dropFileItemsKeyedByIsAbsent[true].Any()) { string missingFiles = string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true].Select(f => $"{f.FullFilePath} ({f})")); return(new IpcResult( IpcResultStatus.InvalidInput, I($"Cannot add the following files to drop because they do not exist:{Environment.NewLine}{missingFiles}"))); } (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync( conf, daemon, directoryPaths, directoryIds, directoryDropPaths, possibleFilters.Result); if (error != null) { return(new IpcResult(IpcResultStatus.ExecutionError, error)); } var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files) if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile)) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"))); } // return early if there is nothing to upload if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any()) { return(new IpcResult(IpcResultStatus.Success, string.Empty)); } return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]))); }