Ejemplo n.º 1
0
        /// <nodoc />
        public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
        {
            Contract.Requires(dropDaemon?.DropConfig != null);

            m_logger             = logger;
            m_dropDaemon         = dropDaemon;
            m_config             = dropDaemon.DropConfig;
            m_cancellationSource = new CancellationTokenSource();

            logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));

            Stats = new DropStatistics();

            // instantiate drop client
            m_dropClient = new ReloadingDropServiceClient(
                logger: logger,
                clientConstructor: CreateDropServiceClient);

            m_nagleQueue = NagleQueue <AddFileItem> .Create(
                maxDegreeOfParallelism : m_config.MaxParallelUploads,
                batchSize : m_config.BatchSize,
                interval : m_config.NagleTime,
                processBatch : ProcessAddFilesAsync);

            if (m_config.ArtifactLogName != null)
            {
                DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
                Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
            }
        }
Ejemplo n.º 2
0
        private static async Task <(IEnumerable <DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
            ConfiguredCommand conf,
            DropDaemon daemon,
            string[] directoryPaths,
            string[] directoryIds,
            string[] dropPaths,
            Regex[] contentFilters)
        {
            Contract.Requires(directoryPaths != null);
            Contract.Requires(directoryIds != null);
            Contract.Requires(dropPaths != null);
            Contract.Requires(contentFilters != null);
            Contract.Requires(directoryPaths.Length == directoryIds.Length);
            Contract.Requires(directoryPaths.Length == dropPaths.Length);
            Contract.Requires(directoryPaths.Length == contentFilters.Length);

            var createDropItemsTasks = Enumerable
                                       .Range(0, directoryPaths.Length)
                                       .Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();

            var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);

            if (createDropItemsResults.Any(r => r.error != null))
            {
                return(null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
            }

            return(createDropItemsResults.SelectMany(r => r.Item1), null);
        }
Ejemplo n.º 3
0
        private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
            DropDaemon daemon,
            string directoryPath,
            string directoryId,
            string dropPath,
            Regex contentFilter)
        {
            Contract.Requires(!string.IsNullOrEmpty(directoryPath));
            Contract.Requires(!string.IsNullOrEmpty(directoryId));
            Contract.Requires(dropPath != null);

            if (daemon.ApiClient == null)
            {
                return(null, "ApiClient is not initialized");
            }

            DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);

            var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);

            if (!maybeResult.Succeeded)
            {
                return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
            }

            var directoryContent = maybeResult.Result;

            daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");

            if (contentFilter != null)
            {
                var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
                daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
                directoryContent = filteredContent;
            }

            List <DropItemForBuildXLFile> dropItemForBuildXLFiles = new List <DropItemForBuildXLFile>();

            var files = directoryContent
                        // SharedOpaque directories might contain 'absent' output files. These are not real files, so we are excluding them.
                        .Where(file => !WellKnownContentHashUtilities.IsAbsentFileHash(file.ContentInfo.Hash) || file.Artifact.IsSourceFile);

            foreach (SealedDirectoryFile file in files)
            {
                // We need to convert '\' into '/' because this path would be a part of a drop url
                // The dropPath can be an empty relative path (i.e. '.') which we need to remove since even though it is not a valid
                // directory name for a Windows file system, it is a valid name for a drop and it doesn't get resolved properly
                var resolvedDropPath = dropPath == "." ? string.Empty : I($"{dropPath}/");
                var remoteFileName   = I($"{resolvedDropPath}{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}");

                dropItemForBuildXLFiles.Add(new DropItemForBuildXLFile(
                                                daemon.ApiClient,
                                                file.FileName,
                                                BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
                                                file.ContentInfo,
                                                remoteFileName));
            }

            return(dropItemForBuildXLFiles.ToArray(), null);
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Uploads the bsi.json for the given drop.
        /// Should be called only when DropConfig.EnableBuildManifestCreation is true.
        /// </summary>
        public async static Task <IIpcResult> UploadBsiFileAsync(DropDaemon daemon)
        {
            Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig");

            if (!System.IO.File.Exists(daemon.DropConfig.BsiFileLocation))
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, $"BuildSessionInfo not found at provided BsiFileLocation: '{daemon.DropConfig.BsiFileLocation}'"));
            }

            var bsiDropItem = new DropItemForFile(daemon.DropConfig.BsiFileLocation, relativeDropPath: DropBsiPath);

            return(await daemon.AddFileAsync(bsiDropItem));
        }
Ejemplo n.º 5
0
        private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
            ConfiguredCommand conf,
            DropDaemon daemon,
            string directoryPath,
            string directoryId,
            string dropPath,
            Regex contentFilter)
        {
            Contract.Requires(!string.IsNullOrEmpty(directoryPath));
            Contract.Requires(!string.IsNullOrEmpty(directoryId));
            Contract.Requires(dropPath != null);

            if (daemon.ApiClient == null)
            {
                return(null, "ApiClient is not initialized");
            }

            DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);

            var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);

            if (!maybeResult.Succeeded)
            {
                return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
            }

            var directoryContent = maybeResult.Result;

            daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");

            if (contentFilter != null)
            {
                var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
                daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
                directoryContent = filteredContent;
            }

            return(directoryContent.Select(file =>
            {
                // we need to convert '\' into '/' because this path would be a part of a drop url
                var remoteFileName = I($"{dropPath}/{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}");

                return new DropItemForBuildXLFile(
                    daemon.ApiClient,
                    file.FileName,
                    BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
                    conf.Get(EnableChunkDedup),
                    file.ContentInfo,
                    remoteFileName);
            }).ToArray(), null);
        }
Ejemplo n.º 6
0
        private static async Task <IIpcResult> AddDropItemsAsync(DropDaemon daemon, IEnumerable <DropItemForBuildXLFile> dropItems)
        {
            (IEnumerable <DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems);

            if (error != null)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, error));
            }

            var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray();
            var ipcResults     = await TaskUtilities.SafeWhenAll(ipcResultTasks);

            return(IpcResult.Merge(ipcResults));
        }
Ejemplo n.º 7
0
        /// <summary>
        /// Generates a BuildManifest.json on the master using all file hashes computed and stored
        /// by workers using <see cref="RegisterFileForBuildManifestAsync"/> for the given drop.
        /// Should be called only when DropConfig.EnableBuildManifestCreation is true.
        /// </summary>
        public async static Task <BuildManifestData> GenerateBuildManifestFileAsync(DropDaemon daemon)
        {
            Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig");

            // TODO: API call returns BuildManifestData, use it to create a local file
            // TODO: Use branch, commitId and Repo, RelativeActivityId/cloudBuildId info available inside dropConfig
            var bxlResult = await daemon.ApiClient.GenerateBuildManifestData(daemon.DropName);

            if (!bxlResult.Succeeded)
            {
                return(new BuildManifestData());
            }

            return(bxlResult.Result);
        }
Ejemplo n.º 8
0
        /// <nodoc />
        public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
        {
            Contract.Requires(dropDaemon?.DropConfig != null);

            m_logger             = logger;
            m_dropDaemon         = dropDaemon;
            m_config             = dropDaemon.DropConfig;
            m_cancellationSource = new CancellationTokenSource();

            logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));

            Stats = new DropStatistics();

            // instantiate drop client
            m_dropClient = new ReloadingDropServiceClient(
                logger: logger,
                clientConstructor: CreateDropServiceClient);

            // create dataflow blocks
            var groupingOptions = new GroupingDataflowBlockOptions {
                Greedy = true
            };

            var actionOptions = new ExecutionDataflowBlockOptions {
                MaxDegreeOfParallelism = m_config.MaxParallelUploads
            };
            var linkOptions = new DataflowLinkOptions {
                PropagateCompletion = true
            };

            m_batchBlock  = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions);
            m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling
            m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions);
            m_batchBlock.LinkTo(m_bufferBlock, linkOptions);
            m_bufferBlock.LinkTo(m_actionBlock, linkOptions);

            // create and set up timer for triggering the batch block
            TimeSpan timerInterval = m_config.NagleTime;

            m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval);

            if (m_config.ArtifactLogName != null)
            {
                DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
                Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
            }
        }
Ejemplo n.º 9
0
        /// <summary>
        /// Takes a hash as string and registers it's corresponding SHA-256 ContentHash using BuildXL Api
        /// Should be called only when DropConfig.EnableBuildManifestCreation is true
        /// </summary>
        private async static Task <bool> RegisterFileForBuildManifestAsync(
            DropDaemon daemon,
            string relativePath,
            ContentHash hash,
            FileArtifact fileId,
            string fullFilePath)
        {
            Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig");
            var bxlResult = await daemon.ApiClient.RegisterFileForBuildManifest(daemon.DropName, relativePath, hash, fileId, fullFilePath);

            if (!bxlResult.Succeeded)
            {
                return(false);
            }

            return(bxlResult.Result);
        }
Ejemplo n.º 10
0
        /// <summary>
        /// Takes a hash as string and registers its corresponding SHA-256 ContentHash using BuildXL Api.
        /// Should be called only when DropConfig.GenerateBuildManifest is true.
        /// Returns a hashset of failing RelativePaths.
        /// </summary>
        private async Task <HashSet <string> > RegisterFilesForBuildManifestAsync(BuildManifestEntry[] buildManifestEntries)
        {
            await Task.Yield();

            Contract.Requires(m_config.GenerateBuildManifest, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig");
            var bxlResult = await m_bxlApiClient.RegisterFilesForBuildManifest(DropDaemon.FullyQualifiedDropName(m_config), buildManifestEntries);

            if (!bxlResult.Succeeded)
            {
                m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest unsuccessful. Failure: {bxlResult.Failure.DescribeIncludingInnerFailures()}");
                return(new HashSet <string>(buildManifestEntries.Select(bme => bme.RelativePath)));
            }

            if (bxlResult.Result.Length > 0)
            {
                m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest found {bxlResult.Result.Length} file hashing failures.");
                return(new HashSet <string>(bxlResult.Result.Select(bme => bme.RelativePath)));
            }

            return(new HashSet <string>());
        }
Ejemplo n.º 11
0
        public static int Main(string[] args)
        {
            // TODO:#1208464- this can be removed once DropDaemon targets .net 4.7 or newer where TLS 1.2 is enabled by default
            ServicePointManager.SecurityProtocol = ServicePointManager.SecurityProtocol | SecurityProtocolType.Tls12;

            try
            {
                Console.WriteLine(nameof(DropDaemon) + " started at " + DateTime.UtcNow);
                Console.WriteLine(DropDaemon.DropDLogPrefix + "Command line arguments: ");
                Console.WriteLine(string.Join(Environment.NewLine + DropDaemon.DropDLogPrefix, args));
                Console.WriteLine();

                DropDaemon.EnsureCommandsInitialized();

                var confCommand = ServicePipDaemon.ServicePipDaemon.ParseArgs(args, new UnixParser());
                if (confCommand.Command.NeedsIpcClient)
                {
                    using (var rpc = CreateClient(confCommand))
                    {
                        var result = confCommand.Command.ClientAction(confCommand, rpc);
                        rpc.RequestStop();
                        rpc.Completion.GetAwaiter().GetResult();
                        return(result);
                    }
                }
                else
                {
                    return(confCommand.Command.ClientAction(confCommand, null));
                }
            }
            catch (ArgumentException e)
            {
                Error(e.Message);
                return(3);
            }
        }
Ejemplo n.º 12
0
        private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
        {
            var files     = File.GetValues(conf.Config).ToArray();
            var fileIds   = FileId.GetValues(conf.Config).ToArray();
            var hashes    = HashOptional.GetValues(conf.Config).ToArray();
            var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();

            if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
            {
                return(new IpcResult(
                           IpcResultStatus.GenericError,
                           I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}")));
            }

            var directoryPaths     = Directory.GetValues(conf.Config).ToArray();
            var directoryIds       = DirectoryId.GetValues(conf.Config).ToArray();
            var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
            var directoryFilters   = DirectoryContentFilter.GetValues(conf.Config).ToArray();

            if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
            {
                return(new IpcResult(
                           IpcResultStatus.GenericError,
                           I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}")));
            }

            (Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
            if (filterInitError != null)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, filterInitError));
            }

            var dropFileItemsKeyedByIsAbsent = Enumerable
                                               .Range(0, files.Length)
                                               .Select(i => new DropItemForBuildXLFile(
                                                           daemon.ApiClient,
                                                           chunkDedup: conf.Get(EnableChunkDedup),
                                                           filePath: files[i],
                                                           fileId: fileIds[i],
                                                           fileContentInfo: FileContentInfo.Parse(hashes[i]),
                                                           relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));

            // If a user specified a particular file to be added to drop, this file must be a part of drop.
            // The missing files will not get into the drop, so we emit an error.
            if (dropFileItemsKeyedByIsAbsent[true].Any())
            {
                return(new IpcResult(
                           IpcResultStatus.InvalidInput,
                           I($"The following files are missing, but they are a part of the drop command:{Environment.NewLine}{string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])}")));
            }

            (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
                conf,
                daemon,
                directoryPaths,
                directoryIds,
                directoryDropPaths,
                initializedFilters);

            if (error != null)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, error));
            }

            var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));

            // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
            if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
            {
                return(new IpcResult(
                           IpcResultStatus.InvalidInput,
                           I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}")));
            }

            // return early if there is nothing to upload
            if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
            {
                return(new IpcResult(IpcResultStatus.Success, string.Empty));
            }

            return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false])));
        }
Ejemplo n.º 13
0
        /// <summary>
        /// Generates and uploads the Manifest.json on the master using all file hashes computed and stored
        /// by workers using <see cref="VsoClient.RegisterFilesForBuildManifestAsync"/> for the given drop.
        /// Should be called only when DropConfig.EnableBuildManifestCreation is true.
        /// </summary>
        public async static Task <IIpcResult> GenerateAndUploadBuildManifestFileAsync(DropDaemon daemon)
        {
            Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig");

            var bxlResult = await daemon.ApiClient.GenerateBuildManifestData(
                daemon.DropName,
                daemon.DropConfig.Repo,
                daemon.DropConfig.Branch,
                daemon.DropConfig.CommitId,
                daemon.DropConfig.CloudBuildId);

            if (!bxlResult.Succeeded)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, $"GenerateBuildManifestData API call failed for Drop: {daemon.DropName}. Failure: {bxlResult.Failure}"));
            }

            string localFilePath;
            string buildManifestJsonStr = BuildManifestData.GenerateBuildManifestJsonString(bxlResult.Result);

            try
            {
                localFilePath = Path.GetTempFileName();
                System.IO.File.WriteAllText(localFilePath, buildManifestJsonStr);
            }
            catch (Exception ex)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, $"Exception while trying to store Build Manifest locally before drop upload: {ex}"));
            }

            var dropItem = new DropItemForFile(localFilePath, relativeDropPath: DropBuildManifestPath);

            return(await daemon.AddFileAsync(dropItem));
        }
Ejemplo n.º 14
0
        private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
        {
            var files     = File.GetValues(conf.Config).ToArray();
            var fileIds   = FileId.GetValues(conf.Config).ToArray();
            var hashes    = HashOptional.GetValues(conf.Config).ToArray();
            var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();

            if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
            {
                return(new IpcResult(
                           IpcResultStatus.GenericError,
                           I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}")));
            }

            var directoryPaths     = Directory.GetValues(conf.Config).ToArray();
            var directoryIds       = DirectoryId.GetValues(conf.Config).ToArray();
            var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
            var directoryFilters   = DirectoryContentFilter.GetValues(conf.Config).ToArray();

            if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
            {
                return(new IpcResult(
                           IpcResultStatus.GenericError,
                           I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}")));
            }

            var possibleFilters = InitializeFilters(directoryFilters);

            if (!possibleFilters.Succeeded)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
            }

            ContentHash[] parsedHashes;

            try
            {
                parsedHashes = hashes.Select(hash => FileContentInfo.Parse(hash).Hash).ToArray();
            }
            catch (ArgumentException e)
            {
                return(new IpcResult(IpcResultStatus.InvalidInput, "Content Hash Parsing exception: " + e.InnerException));
            }

            if (daemon.DropConfig.EnableBuildManifestCreation)
            {
                var buildManifestHashTasks = Enumerable
                                             .Range(0, parsedHashes.Length)
                                             .Select(i => RegisterFileForBuildManifestAsync(daemon, dropPaths[i], parsedHashes[i], BuildXL.Ipc.ExternalApi.FileId.Parse(fileIds[i]), files[i]));

                var buildManifestHashes = await TaskUtilities.SafeWhenAll(buildManifestHashTasks);

                if (buildManifestHashes.Any(h => h == false))
                {
                    return(new IpcResult(IpcResultStatus.ExecutionError, "Failure during BuildManifest Hash generation"));
                }
            }

            var dropFileItemsKeyedByIsAbsent = Enumerable
                                               .Range(0, files.Length)
                                               .Select(i => new DropItemForBuildXLFile(
                                                           daemon.ApiClient,
                                                           filePath: files[i],
                                                           fileId: fileIds[i],
                                                           fileContentInfo: FileContentInfo.Parse(hashes[i]),
                                                           relativeDropPath: dropPaths[i]))
                                               .ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));

            // If a user specified a particular file to be added to drop, this file must be a part of drop.
            // The missing files will not get into the drop, so we emit an error.
            if (dropFileItemsKeyedByIsAbsent[true].Any())
            {
                string missingFiles = string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true].Select(f => $"{f.FullFilePath} ({f})"));
                return(new IpcResult(
                           IpcResultStatus.InvalidInput,
                           I($"Cannot add the following files to drop because they do not exist:{Environment.NewLine}{missingFiles}")));
            }

            (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
                conf,
                daemon,
                directoryPaths,
                directoryIds,
                directoryDropPaths,
                possibleFilters.Result);

            if (error != null)
            {
                return(new IpcResult(IpcResultStatus.ExecutionError, error));
            }

            var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));

            // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
            if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
            {
                return(new IpcResult(
                           IpcResultStatus.InvalidInput,
                           I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}")));
            }

            // return early if there is nothing to upload
            if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
            {
                return(new IpcResult(IpcResultStatus.Success, string.Empty));
            }

            return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false])));
        }