private static async Task <(IEnumerable <DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
ConfiguredCommand conf,
Daemon daemon,
string[] directoryPaths,
string[] directoryIds,
string[] dropPaths,
Regex[] contentFilters)
{
Contract.Requires(directoryPaths != null);
Contract.Requires(directoryIds != null);
Contract.Requires(dropPaths != null);
Contract.Requires(contentFilters != null);
Contract.Requires(directoryPaths.Length == directoryIds.Length);
Contract.Requires(directoryPaths.Length == dropPaths.Length);
Contract.Requires(directoryPaths.Length == contentFilters.Length);
var createDropItemsTasks = Enumerable
.Range(0, directoryPaths.Length)
.Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();
var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);
if (createDropItemsResults.Any(r => r.error != null))
{
return(null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
}
return(createDropItemsResults.SelectMany(r => r.Item1), null);
}
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
{
byte?domainId;
checked
{
domainId = (byte?)conf.Get(OptionalDropDomainId);
}
return(new DropConfig(
dropName: conf.Get(DropNameOption),
serviceEndpoint: conf.Get(DropEndpoint),
maxParallelUploads: conf.Get(MaxParallelUploads),
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
verbose: conf.Get(Verbose),
enableTelemetry: conf.Get(EnableTelemetry),
enableChunkDedup: conf.Get(EnableChunkDedup),
logDir: conf.Get(LogDir),
artifactLogName: conf.Get(ArtifactLogName),
batchSize: conf.Get(BatchSize),
dropDomainId: domainId,
enableBuildManifestCreation: conf.Get(EnableBuildManifestCreation),
repo: conf.Get(Repo),
branch: conf.Get(Branch),
commitId: conf.Get(CommitId),
cloudBuildId: conf.Get(CloudBuildId),
bsiFileLocation: conf.Get(BsiFileLocation)));
}
private static IIpcResult RPCSendCore(ConfiguredCommand conf, IClient rpc, bool isSync)
{
string operationPayload = ToPayload(conf);
var operation = new IpcOperation(operationPayload, waitForServerAck: isSync);
return(rpc.Send(operation).GetAwaiter().GetResult());
}
private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
ConfiguredCommand conf,
Daemon daemon,
string directoryPath,
string directoryId,
string dropPath,
Regex contentFilter)
{
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
Contract.Requires(!string.IsNullOrEmpty(directoryId));
Contract.Requires(dropPath != null);
// TODO: can be removed when the IPC lazy directory dependencies feature is implemented
if (!System.IO.Directory.Exists(directoryPath))
{
return(null, Inv("directory '{0}' does not exist", directoryPath));
}
if (daemon.ApiClient == null)
{
return(null, "ApiClient is not initialized");
}
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
if (!maybeResult.Succeeded)
{
return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
}
var directoryContent = maybeResult.Result;
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
if (contentFilter != null)
{
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
directoryContent = filteredContent;
}
return(directoryContent.Select(file =>
{
var remoteFileName = Inv(
"{0}/{1}",
dropPath,
// we need to convert '\' into '/' because this path would be a part of a drop url
GetRelativePath(directoryPath, file.FileName).Replace('\\', '/'));
return new DropItemForBuildXLFile(
daemon.ApiClient,
file.FileName,
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
conf.Get(EnableChunkDedup),
file.ContentInfo,
remoteFileName);
}).ToArray(), null);
}
private static async Task <(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
ConfiguredCommand conf,
DropDaemon daemon,
string directoryPath,
string directoryId,
string dropPath,
Regex contentFilter)
{
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
Contract.Requires(!string.IsNullOrEmpty(directoryId));
Contract.Requires(dropPath != null);
if (daemon.ApiClient == null)
{
return(null, "ApiClient is not initialized");
}
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
if (!maybeResult.Succeeded)
{
return(null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
}
var directoryContent = maybeResult.Result;
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
if (contentFilter != null)
{
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
directoryContent = filteredContent;
}
return(directoryContent
// SharedOpaque directories might contain 'absent' output files. These are not real files, so we are excluding them.
.Where(file => !WellKnownContentHashUtilities.IsAbsentFileHash(file.ContentInfo.Hash) || file.Artifact.IsSourceFile)
.Select(file =>
{
// We need to convert '\' into '/' because this path would be a part of a drop url
// The dropPath can be an empty relative path (i.e. '.') which we need to remove since even though it is not a valid
// directory name for a Windows file system, it is a valid name for a drop and it doesn't get resolved properly
var resolvedDropPath = dropPath == "." ? string.Empty : I($"{dropPath}/");
var remoteFileName = I($"{resolvedDropPath}{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}");
return new DropItemForBuildXLFile(
daemon.ApiClient,
file.FileName,
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
file.ContentInfo,
remoteFileName);
}).ToArray(), null);
}
internal static DaemonConfig CreateDaemonConfig(ConfiguredCommand conf)
{
return(new DaemonConfig(
logger: conf.Logger,
moniker: conf.Get(Moniker),
maxConnectRetries: conf.Get(MaxConnectRetries),
connectRetryDelay: TimeSpan.FromMilliseconds(conf.Get(ConnectRetryDelayMillis)),
stopOnFirstFailure: conf.Get(StopOnFirstFailure),
enableCloudBuildIntegration: conf.Get(EnableCloudBuildIntegration)));
}
private static int RPCSend(ConfiguredCommand conf, IClient rpc, bool isSync)
{
var rpcResult = RPCSendCore(conf, rpc, isSync);
conf.Logger.Info(
"Command '{0}' {1} (exit code: {2}). {3}",
conf.Command.Name,
rpcResult.Succeeded ? "succeeded" : "failed",
(int)rpcResult.ExitCode,
rpcResult.Payload);
return((int)rpcResult.ExitCode);
}
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
{
return(new DropConfig(
dropName: conf.Get(DropName),
serviceEndpoint: conf.Get(DropEndpoint),
maxParallelUploads: conf.Get(MaxParallelUploads),
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
verbose: conf.Get(Verbose),
enableTelemetry: conf.Get(EnableTelemetry),
enableChunkDedup: conf.Get(EnableChunkDedup),
logDir: conf.Get(LogDir)));
}
internal static bool VerifyBuildManifestRequirements(ConfiguredCommand conf, DropConfig dropConfig)
{
if (dropConfig.EnableBuildManifestCreation == true)
{
List <string> missingFields = new List <string>();
if (string.IsNullOrEmpty(dropConfig.Repo))
{
missingFields.Add("repo");
}
if (string.IsNullOrEmpty(dropConfig.Branch))
{
missingFields.Add("branch");
}
if (string.IsNullOrEmpty(dropConfig.CommitId))
{
missingFields.Add("commitId");
}
if (string.IsNullOrEmpty(dropConfig.CloudBuildId))
{
missingFields.Add("cloudBuildId");
}
if (string.IsNullOrEmpty(dropConfig.BsiFileLocation))
{
missingFields.Add("BsiFileLocation");
}
if (missingFields.Count != 0)
{
conf.Logger.Error($"EnableBuildManifestCreation set to true, but the following required fields are missing: {string.Join(", ", missingFields)}");
return(false);
}
}
return(true);
}
public static int Main(string[] args)
{
// TODO:# 1208464- this can be removed once DropDaemon targets .net or newer 4.7 where TLS 1.2 is enabled by default
ServicePointManager.SecurityProtocol = ServicePointManager.SecurityProtocol | SecurityProtocolType.Tls12;
if (args.Length > 0 && args[0] == "listen")
{
return(SubscribeAndProcessCloudBuildEvents());
}
try
{
Console.WriteLine("DropDaemon started at " + DateTime.UtcNow);
Console.WriteLine(Daemon.DropDLogPrefix + "Command line arguments: ");
Console.WriteLine(string.Join(Environment.NewLine + Daemon.DropDLogPrefix, args));
Console.WriteLine();
ConfiguredCommand conf = ParseArgs(args, new UnixParser());
if (conf.Command.NeedsIpcClient)
{
using (var rpc = CreateClient(conf))
{
var result = conf.Command.ClientAction(conf, rpc);
rpc.RequestStop();
rpc.Completion.GetAwaiter().GetResult();
return(result);
}
}
else
{
return(conf.Command.ClientAction(conf, null));
}
}
catch (ArgumentException e)
{
Error(e.Message);
return(3);
}
}
/// <summary> /// Reconstructs a full command line corresponding to a <see cref="ConfiguredCommand"/>. /// </summary> private static string ToPayload(ConfiguredCommand cmd) => ToPayload(cmd.Command.Name, cmd.Config);
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
{
var files = File.GetValues(conf.Config).ToArray();
var fileIds = FileId.GetValues(conf.Config).ToArray();
var hashes = HashOptional.GetValues(conf.Config).ToArray();
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}")));
}
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}")));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
ContentHash[] parsedHashes;
try
{
parsedHashes = hashes.Select(hash => FileContentInfo.Parse(hash).Hash).ToArray();
}
catch (ArgumentException e)
{
return(new IpcResult(IpcResultStatus.InvalidInput, "Content Hash Parsing exception: " + e.InnerException));
}
if (daemon.DropConfig.EnableBuildManifestCreation)
{
var buildManifestHashTasks = Enumerable
.Range(0, parsedHashes.Length)
.Select(i => RegisterFileForBuildManifestAsync(daemon, dropPaths[i], parsedHashes[i], BuildXL.Ipc.ExternalApi.FileId.Parse(fileIds[i]), files[i]));
var buildManifestHashes = await TaskUtilities.SafeWhenAll(buildManifestHashTasks);
if (buildManifestHashes.Any(h => h == false))
{
return(new IpcResult(IpcResultStatus.ExecutionError, "Failure during BuildManifest Hash generation"));
}
}
var dropFileItemsKeyedByIsAbsent = Enumerable
.Range(0, files.Length)
.Select(i => new DropItemForBuildXLFile(
daemon.ApiClient,
filePath: files[i],
fileId: fileIds[i],
fileContentInfo: FileContentInfo.Parse(hashes[i]),
relativeDropPath: dropPaths[i]))
.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// If a user specified a particular file to be added to drop, this file must be a part of drop.
// The missing files will not get into the drop, so we emit an error.
if (dropFileItemsKeyedByIsAbsent[true].Any())
{
string missingFiles = string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true].Select(f => $"{f.FullFilePath} ({f})"));
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Cannot add the following files to drop because they do not exist:{Environment.NewLine}{missingFiles}")));
}
(IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
conf,
daemon,
directoryPaths,
directoryIds,
directoryDropPaths,
possibleFilters.Result);
if (error != null)
{
return(new IpcResult(IpcResultStatus.ExecutionError, error));
}
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}")));
}
// return early if there is nothing to upload
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
{
return(new IpcResult(IpcResultStatus.Success, string.Empty));
}
return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false])));
}
private static int AsyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, false);
private static int SyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, true);
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
return(IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig));
}
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = CreateDaemonConfig(conf);
return(IpcProvider.GetClient(daemonConfig.Moniker, daemonConfig));
}
private static async Task <IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, Daemon daemon)
{
var files = File.GetValues(conf.Config).ToArray();
var fileIds = FileId.GetValues(conf.Config).ToArray();
var hashes = HashOptional.GetValues(conf.Config).ToArray();
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
Inv(
"File counts don't match: #files = {0}, #fileIds = {1}, #hashes = {2}, #dropPaths = {3}",
files.Length, fileIds.Length, hashes.Length, dropPaths.Length)));
}
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
Inv(
"Directory counts don't match: #directories = {0}, #directoryIds = {1}, #dropPaths = {2}, #directoryFilters = {3}",
directoryPaths.Length, directoryIds.Length, directoryDropPaths.Length, directoryFilters.Length)));
}
(Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
if (filterInitError != null)
{
return(new IpcResult(IpcResultStatus.ExecutionError, filterInitError));
}
var dropFileItemsKeyedByIsAbsent = Enumerable
.Range(0, files.Length)
.Select(i => new DropItemForBuildXLFile(
daemon.ApiClient,
chunkDedup: conf.Get(EnableChunkDedup),
filePath: files[i],
fileId: fileIds[i],
fileContentInfo: FileContentInfo.Parse(hashes[i]),
relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// If a user specified a particular file to be added to drop, this file must be a part of drop.
// The missing files will not get into the drop, so we emit an error.
if (dropFileItemsKeyedByIsAbsent[true].Any())
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
Inv("The following files are missing, but they are a part of the drop command:{0}{1}",
Environment.NewLine,
string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true]))));
}
(IEnumerable <DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
conf,
daemon,
directoryPaths,
directoryIds,
directoryDropPaths,
initializedFilters);
if (error != null)
{
return(new IpcResult(IpcResultStatus.ExecutionError, error));
}
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
Inv("Uploading missing source file(s) is not supported:{0}{1}",
Environment.NewLine,
string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile)))));
}
// return early if there is nothing to upload
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
{
return(new IpcResult(IpcResultStatus.Success, string.Empty));
}
return(await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false])));
}
