/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); m_nagleQueue = NagleQueue <AddFileItem> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessAddFilesAsync); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
private static async Task <(IEnumerable <DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync( ConfiguredCommand conf, DropDaemon daemon, string[] directoryPaths, string[] directoryIds, string[] dropPaths, Regex[] contentFilters) { Contract.Requires(directoryPaths != null); Contract.Requires(directoryIds != null); Contract.Requires(dropPaths != null); Contract.Requires(contentFilters != null); Contract.Requires(directoryPaths.Length == directoryIds.Length); Contract.Requires(directoryPaths.Length == dropPaths.Length); Contract.Requires(directoryPaths.Length == contentFilters.Length); var createDropItemsTasks = Enumerable .Range(0, directoryPaths.Length) .Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray(); var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks); if (createDropItemsResults.Any(r => r.error != null)) { return(null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error))); } return(createDropItemsResults.SelectMany(r => r.Item1), null); }
private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync( DropDaemon daemon, string directoryPath, string directoryId, string dropPath, Regex contentFilter) { Contract.Requires(!string.IsNullOrEmpty(directoryPath)); Contract.Requires(!string.IsNullOrEmpty(directoryId)); Contract.Requires(dropPath != null); if (daemon.ApiClient == null) { return(null, "ApiClient is not initialized"); } DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId); var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath); if (!maybeResult.Succeeded) { return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe()); } var directoryContent = maybeResult.Result; daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}"); if (contentFilter != null) { var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList(); daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count); directoryContent = filteredContent; } List <DropItemForBuildXLFile> dropItemForBuildXLFiles = new List <DropItemForBuildXLFile>(); var files = directoryContent // SharedOpaque directories might contain 'absent' output files. These are not real files, so we are excluding them. .Where(file => !WellKnownContentHashUtilities.IsAbsentFileHash(file.ContentInfo.Hash) || file.Artifact.IsSourceFile); foreach (SealedDirectoryFile file in files) { // We need to convert '\' into '/' because this path would be a part of a drop url // The dropPath can be an empty relative path (i.e. '.') which we need to remove since even though it is not a valid // directory name for a Windows file system, it is a valid name for a drop and it doesn't get resolved properly var resolvedDropPath = dropPath == "." ? string.Empty : I($"{dropPath}/"); var remoteFileName = I($"{resolvedDropPath}{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}"); dropItemForBuildXLFiles.Add(new DropItemForBuildXLFile( daemon.ApiClient, file.FileName, BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact), file.ContentInfo, remoteFileName)); } return(dropItemForBuildXLFiles.ToArray(), null); }
/// <summary> /// Uploads the bsi.json for the given drop. /// Should be called only when DropConfig.EnableBuildManifestCreation is true. /// </summary> public async static Task <IIpcResult> UploadBsiFileAsync(DropDaemon daemon) { Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig"); if (!System.IO.File.Exists(daemon.DropConfig.BsiFileLocation)) { return(new IpcResult(IpcResultStatus.ExecutionError, $"BuildSessionInfo not found at provided BsiFileLocation: '{daemon.DropConfig.BsiFileLocation}'")); } var bsiDropItem = new DropItemForFile(daemon.DropConfig.BsiFileLocation, relativeDropPath: DropBsiPath); return(await daemon.AddFileAsync(bsiDropItem)); }
private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync( ConfiguredCommand conf, DropDaemon daemon, string directoryPath, string directoryId, string dropPath, Regex contentFilter) { Contract.Requires(!string.IsNullOrEmpty(directoryPath)); Contract.Requires(!string.IsNullOrEmpty(directoryId)); Contract.Requires(dropPath != null); if (daemon.ApiClient == null) { return(null, "ApiClient is not initialized"); } DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId); var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath); if (!maybeResult.Succeeded) { return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe()); } var directoryContent = maybeResult.Result; daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}"); if (contentFilter != null) { var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList(); daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count); directoryContent = filteredContent; } return(directoryContent.Select(file => { // we need to convert '\' into '/' because this path would be a part of a drop url var remoteFileName = I($"{dropPath}/{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}"); return new DropItemForBuildXLFile( daemon.ApiClient, file.FileName, BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact), conf.Get(EnableChunkDedup), file.ContentInfo, remoteFileName); }).ToArray(), null); }
private static async Task <IIpcResult> AddDropItemsAsync(DropDaemon daemon, IEnumerable <DropItemForBuildXLFile> dropItems) { (IEnumerable <DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems); if (error != null) { return(new IpcResult(IpcResultStatus.ExecutionError, error)); } var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray(); var ipcResults = await TaskUtilities.SafeWhenAll(ipcResultTasks); return(IpcResult.Merge(ipcResults)); }
/// <summary> /// Generates a BuildManifest.json on the master using all file hashes computed and stored /// by workers using <see cref="RegisterFileForBuildManifestAsync"/> for the given drop. /// Should be called only when DropConfig.EnableBuildManifestCreation is true. /// </summary> public async static Task <BuildManifestData> GenerateBuildManifestFileAsync(DropDaemon daemon) { Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig"); // TODO: API call returns BuildManifestData, use it to create a local file // TODO: Use branch, commitId and Repo, RelativeActivityId/cloudBuildId info available inside dropConfig var bxlResult = await daemon.ApiClient.GenerateBuildManifestData(daemon.DropName); if (!bxlResult.Succeeded) { return(new BuildManifestData()); } return(bxlResult.Result); }
/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); // create dataflow blocks var groupingOptions = new GroupingDataflowBlockOptions { Greedy = true }; var actionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = m_config.MaxParallelUploads }; var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions); m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions); m_batchBlock.LinkTo(m_bufferBlock, linkOptions); m_bufferBlock.LinkTo(m_actionBlock, linkOptions); // create and set up timer for triggering the batch block TimeSpan timerInterval = m_config.NagleTime; m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
/// <summary> /// Takes a hash as string and registers it's corresponding SHA-256 ContentHash using BuildXL Api /// Should be called only when DropConfig.EnableBuildManifestCreation is true /// </summary> private async static Task <bool> RegisterFileForBuildManifestAsync( DropDaemon daemon, string relativePath, ContentHash hash, FileArtifact fileId, string fullFilePath) { Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig"); var bxlResult = await daemon.ApiClient.RegisterFileForBuildManifest(daemon.DropName, relativePath, hash, fileId, fullFilePath); if (!bxlResult.Succeeded) { return(false); } return(bxlResult.Result); }
/// <summary> /// Takes a hash as string and registers its corresponding SHA-256 ContentHash using BuildXL Api. /// Should be called only when DropConfig.GenerateBuildManifest is true. /// Returns a hashset of failing RelativePaths. /// </summary> private async Task <HashSet <string> > RegisterFilesForBuildManifestAsync(BuildManifestEntry[] buildManifestEntries) { await Task.Yield(); Contract.Requires(m_config.GenerateBuildManifest, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig"); var bxlResult = await m_bxlApiClient.RegisterFilesForBuildManifest(DropDaemon.FullyQualifiedDropName(m_config), buildManifestEntries); if (!bxlResult.Succeeded) { m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest unsuccessful. Failure: {bxlResult.Failure.DescribeIncludingInnerFailures()}"); return(new HashSet <string>(buildManifestEntries.Select(bme => bme.RelativePath))); } if (bxlResult.Result.Length > 0) { m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest found {bxlResult.Result.Length} file hashing failures."); return(new HashSet <string>(bxlResult.Result.Select(bme => bme.RelativePath))); } return(new HashSet <string>()); }
public static int Main(string[] args) { // TODO:#1208464- this can be removed once DropDaemon targets .net 4.7 or newer where TLS 1.2 is enabled by default ServicePointManager.SecurityProtocol = ServicePointManager.SecurityProtocol | SecurityProtocolType.Tls12; try { Console.WriteLine(nameof(DropDaemon) + " started at " + DateTime.UtcNow); Console.WriteLine(DropDaemon.DropDLogPrefix + "Command line arguments: "); Console.WriteLine(string.Join(Environment.NewLine + DropDaemon.DropDLogPrefix, args)); Console.WriteLine(); DropDaemon.EnsureCommandsInitialized(); var confCommand = ServicePipDaemon.ServicePipDaemon.ParseArgs(args, new UnixParser()); if (confCommand.Command.NeedsIpcClient) { using (var rpc = CreateClient(confCommand)) { var result = confCommand.Command.ClientAction(confCommand, rpc); rpc.RequestStop(); rpc.Completion.GetAwaiter().GetResult(); return(result); } } else { return(confCommand.Command.ClientAction(confCommand, null)); } } catch (ArgumentException e) { Error(e.Message); return(3); } }
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon) { var files = File.GetValues(conf.Config).ToArray(); var fileIds = FileId.GetValues(conf.Config).ToArray(); var hashes = HashOptional.GetValues(conf.Config).ToArray(); var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray(); if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"))); } var directoryPaths = Directory.GetValues(conf.Config).ToArray(); var directoryIds = DirectoryId.GetValues(conf.Config).ToArray(); var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray(); var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray(); if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"))); } (Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters); if (filterInitError != null) { return(new IpcResult(IpcResultStatus.ExecutionError, filterInitError)); } var dropFileItemsKeyedByIsAbsent = Enumerable .Range(0, files.Length) .Select(i => new DropItemForBuildXLFile( daemon.ApiClient, chunkDedup: conf.Get(EnableChunkDedup), filePath: files[i], fileId: fileIds[i], fileContentInfo: FileContentInfo.Parse(hashes[i]), relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // If a user specified a particular file to be added to drop, this file must be a part of drop. // The missing files will not get into the drop, so we emit an error. if (dropFileItemsKeyedByIsAbsent[true].Any()) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"The following files are missing, but they are a part of the drop command:{Environment.NewLine}{string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])}"))); } (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync( conf, daemon, directoryPaths, directoryIds, directoryDropPaths, initializedFilters); if (error != null) { return(new IpcResult(IpcResultStatus.ExecutionError, error)); } var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files) if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile)) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"))); } // return early if there is nothing to upload if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any()) { return(new IpcResult(IpcResultStatus.Success, string.Empty)); } return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]))); }
/// <summary> /// Generates and uploads the Manifest.json on the master using all file hashes computed and stored /// by workers using <see cref="VsoClient.RegisterFilesForBuildManifestAsync"/> for the given drop. /// Should be called only when DropConfig.EnableBuildManifestCreation is true. /// </summary> public async static Task <IIpcResult> GenerateAndUploadBuildManifestFileAsync(DropDaemon daemon) { Contract.Requires(daemon.DropConfig.EnableBuildManifestCreation == true, "GenerateBuildManifestData API called even though Build Manifest Generation is Disabled in DropConfig"); var bxlResult = await daemon.ApiClient.GenerateBuildManifestData( daemon.DropName, daemon.DropConfig.Repo, daemon.DropConfig.Branch, daemon.DropConfig.CommitId, daemon.DropConfig.CloudBuildId); if (!bxlResult.Succeeded) { return(new IpcResult(IpcResultStatus.ExecutionError, $"GenerateBuildManifestData API call failed for Drop: {daemon.DropName}. Failure: {bxlResult.Failure}")); } string localFilePath; string buildManifestJsonStr = BuildManifestData.GenerateBuildManifestJsonString(bxlResult.Result); try { localFilePath = Path.GetTempFileName(); System.IO.File.WriteAllText(localFilePath, buildManifestJsonStr); } catch (Exception ex) { return(new IpcResult(IpcResultStatus.ExecutionError, $"Exception while trying to store Build Manifest locally before drop upload: {ex}")); } var dropItem = new DropItemForFile(localFilePath, relativeDropPath: DropBuildManifestPath); return(await daemon.AddFileAsync(dropItem)); }
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon) { var files = File.GetValues(conf.Config).ToArray(); var fileIds = FileId.GetValues(conf.Config).ToArray(); var hashes = HashOptional.GetValues(conf.Config).ToArray(); var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray(); if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"))); } var directoryPaths = Directory.GetValues(conf.Config).ToArray(); var directoryIds = DirectoryId.GetValues(conf.Config).ToArray(); var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray(); var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray(); if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length) { return(new IpcResult( IpcResultStatus.GenericError, I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"))); } var possibleFilters = InitializeFilters(directoryFilters); if (!possibleFilters.Succeeded) { return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe())); } ContentHash[] parsedHashes; try { parsedHashes = hashes.Select(hash => FileContentInfo.Parse(hash).Hash).ToArray(); } catch (ArgumentException e) { return(new IpcResult(IpcResultStatus.InvalidInput, "Content Hash Parsing exception: " + e.InnerException)); } if (daemon.DropConfig.EnableBuildManifestCreation) { var buildManifestHashTasks = Enumerable .Range(0, parsedHashes.Length) .Select(i => RegisterFileForBuildManifestAsync(daemon, dropPaths[i], parsedHashes[i], BuildXL.Ipc.ExternalApi.FileId.Parse(fileIds[i]), files[i])); var buildManifestHashes = await TaskUtilities.SafeWhenAll(buildManifestHashTasks); if (buildManifestHashes.Any(h => h == false)) { return(new IpcResult(IpcResultStatus.ExecutionError, "Failure during BuildManifest Hash generation")); } } var dropFileItemsKeyedByIsAbsent = Enumerable .Range(0, files.Length) .Select(i => new DropItemForBuildXLFile( daemon.ApiClient, filePath: files[i], fileId: fileIds[i], fileContentInfo: FileContentInfo.Parse(hashes[i]), relativeDropPath: dropPaths[i])) .ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // If a user specified a particular file to be added to drop, this file must be a part of drop. // The missing files will not get into the drop, so we emit an error. if (dropFileItemsKeyedByIsAbsent[true].Any()) { string missingFiles = string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true].Select(f => $"{f.FullFilePath} ({f})")); return(new IpcResult( IpcResultStatus.InvalidInput, I($"Cannot add the following files to drop because they do not exist:{Environment.NewLine}{missingFiles}"))); } (IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync( conf, daemon, directoryPaths, directoryIds, directoryDropPaths, possibleFilters.Result); if (error != null) { return(new IpcResult(IpcResultStatus.ExecutionError, error)); } var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash)); // we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files) if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile)) { return(new IpcResult( IpcResultStatus.InvalidInput, I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"))); } // return early if there is nothing to upload if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any()) { return(new IpcResult(IpcResultStatus.Success, string.Empty)); } return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]))); }