private static CommandInfo Build(CommandInfo parent, ConfiguredCommand command)
{
var info = new CommandInfo(parent, command);
foreach (var parameter in GetParameters(info))
{
info.Parameters.Add(parameter);
}
foreach (var childCommand in command.Children)
{
var child = Build(info, childCommand);
info.Children.Add(child);
}
// Normalize argument positions.
var index = 0;
foreach (var argument in info.Parameters.OfType <CommandArgument>()
.OrderBy(argument => argument.Position))
{
argument.Position = index;
index++;
}
return(info);
}
public CommandInfo(CommandInfo?parent, ConfiguredCommand prototype)
{
Parent = parent;
Name = prototype.Name;
Aliases = new HashSet <string>(prototype.Aliases);
Description = prototype.Description;
Data = prototype.Data;
CommandType = prototype.CommandType;
SettingsType = prototype.SettingsType;
Delegate = prototype.Delegate;
IsDefaultCommand = prototype.IsDefaultCommand;
IsHidden = prototype.IsHidden;
Children = new List <CommandInfo>();
Parameters = new List <CommandParameter>();
Examples = prototype.Examples ?? new List <string[]>();
if (CommandType != null && string.IsNullOrWhiteSpace(Description))
{
var description = CommandType.GetCustomAttribute <DescriptionAttribute>();
if (description != null)
{
Description = description.Description;
}
}
}
public ICommandConfigurator AddCommand <TCommand>(string name)
where TCommand : class, ICommand
{
var command = Commands.AddAndReturn(ConfiguredCommand.FromType <TCommand>(name, false));
return(new CommandConfigurator(command));
}
public ICommandConfigurator AddDelegate <TSettings>(string name, Func <CommandContext, TSettings, int> func)
where TSettings : CommandSettings
{
var command = Commands.AddAndReturn(ConfiguredCommand.FromDelegate <TSettings>(
name, (context, settings) => func(context, (TSettings)settings)));
return(new CommandConfigurator(command));
}
public void AddBranch <TSettings>(string name, Action <IConfigurator <TSettings> > action)
where TSettings : CommandSettings
{
var command = ConfiguredCommand.FromBranch <TSettings>(name);
action(new Configurator <TSettings>(command, _registrar));
Commands.Add(command);
}
public void AddBranch <TDerivedSettings>(string name, Action <IConfigurator <TDerivedSettings> > action)
where TDerivedSettings : TSettings
{
var command = ConfiguredCommand.FromBranch <TDerivedSettings>(name);
action(new Configurator <TDerivedSettings>(command, _registrar));
_command.Children.Add(command);
}
public ICommandConfigurator AddDelegate <TDerivedSettings>(string name, Func <CommandContext, TDerivedSettings, int> func)
where TDerivedSettings : TSettings
{
var command = ConfiguredCommand.FromDelegate <TDerivedSettings>(
name, (context, settings) => func(context, (TDerivedSettings)settings));
_command.Children.Add(command);
return(new CommandConfigurator(command));
}
public ICommandConfigurator AddCommand <TCommand>(string name)
where TCommand : class, ICommandLimiter <TSettings>
{
var command = ConfiguredCommand.FromType <TCommand>(name);
var configurator = new CommandConfigurator(command);
_command.Children.Add(command);
return(configurator);
}
public CommandInfo(CommandInfo parent, ConfiguredCommand prototype)
{
Parent = parent;
Name = prototype.Name;
Description = prototype.Description;
CommandType = prototype.CommandType;
SettingsType = prototype.SettingsType;
Children = new List <CommandInfo>();
Parameters = new List <CommandParameter>();
}
public void TestConnectionErrorExitCode()
{
ConfiguredCommand conf = ParseArgs($"{StopDaemonCmd.Name} --{Moniker.LongName} 1 --{ConnectRetryDelayMillis.LongName} 10 --{MaxConnectRetries.LongName} 0");
using (var client = CreateClient(conf))
{
var exitCode = conf.Command.ClientAction(conf, client);
Assert.Equal((int)IpcResultStatus.ConnectionError, exitCode);
client.RequestStop();
client.Completion.GetAwaiter().GetResult();
}
}
void IUnsafeConfigurator.AddBranch(string name, Type settings, Action <IUnsafeBranchConfigurator> action)
{
var command = ConfiguredCommand.FromBranch(settings, name);
// Create the configurator.
var configuratorType = typeof(Configurator <>).MakeGenericType(settings);
if (!(Activator.CreateInstance(configuratorType, new object?[] { command, _registrar }) is IUnsafeBranchConfigurator configurator))
{
throw new CommandConfigurationException("Could not create configurator by reflection.");
}
action(configurator);
Commands.Add(command);
}
private Thread CreateThreadForCommand(string cmdLine, IClient client)
{
return(new Thread(() =>
{
Console.WriteLine($"running command: " + cmdLine);
var logger = new LambdaLogger((level, format, args) =>
{
var formatted = LoggerExtensions.Format(level, format, args);
Console.WriteLine(format);
Output.WriteLine(format);
});
ConfiguredCommand conf = ParseArgs(cmdLine, UnixParser.Instance, logger);
var exitCode = conf.Command.ClientAction(conf, client);
Assert.Equal(0, exitCode);
}));
}
private static async Task <IIpcResult> RegisterManifestInternalAsync(ConfiguredCommand conf, MaterializationDaemon daemon)
{
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}")));
}
if (daemon.ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var manifests = new List <SealedDirectoryFile>();
for (int i = 0; i < directoryIds.Length; i++)
{
var directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryIds[i]);
var possibleContent = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPaths[i]);
if (!possibleContent.Succeeded)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Failed to get the content of a directory artifact ({directoryIds[i]}, {directoryPaths[i]}){Environment.NewLine}{possibleContent.Failure.DescribeIncludingInnerFailures()}")));
}
manifests.AddRange(possibleContent.Result);
}
daemon.Logger.Verbose(string.Join(Environment.NewLine, manifests.Select(f => f.FileName)));
// TODO: placeholder for now - to be implemented in another pr
return(IpcResult.Success("done"));
}
public CommandInfo(CommandInfo parent, ConfiguredCommand prototype)
{
Parent = parent;
Name = prototype.Name;
Description = prototype.Description;
CommandType = prototype.CommandType;
SettingsType = prototype.SettingsType;
IsDefaultCommand = prototype.IsDefaultCommand;
Children = new List <CommandInfo>();
Parameters = new List <CommandParameter>();
Examples = prototype.Examples ?? new List <string[]>();
if (!IsBranch)
{
var description = CommandType.GetCustomAttribute <DescriptionAttribute>();
if (description != null)
{
Description = description.Description;
}
}
}
private static CommandInfo Build(CommandInfo parent, ConfiguredCommand command)
{
var info = new CommandInfo(parent, command);
if (!info.IsProxy)
{
var description = info.CommandType.GetCustomAttribute <DescriptionAttribute>();
if (description != null)
{
info.Description = description.Description;
}
}
foreach (var parameter in GetParameters(info))
{
info.Parameters.Add(parameter);
}
foreach (var childCommand in command.Children)
{
var child = Build(info, childCommand);
info.Children.Add(child);
}
// Normalize argument positions.
var index = 0;
foreach (var argument in info.Parameters.OfType <CommandArgument>()
.OrderBy(argument => argument.Position))
{
argument.Position = index;
index++;
}
return(info);
}
internal static MaterializationDaemonConfig CreateMaterializationDaemonConfig(ConfiguredCommand conf)
{
return(new MaterializationDaemonConfig(
maxDegreeOfParallelism: conf.Get(MaxDegreeOfParallelism),
parserExe: conf.Get(ManifestParserExeLocation),
parserArgs: conf.Get(ManifestParserAdditionalArgs),
logDir: conf.Get(LogDir)));
}
public Configurator(ConfiguredCommand command, ITypeRegistrar?registrar)
{
_command = command;
_registrar = registrar;
}
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
return(IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig));
}
public void SetDefaultCommand <TDefaultCommand>()
where TDefaultCommand : class, ICommand
{
DefaultCommand = ConfiguredCommand.FromType <TDefaultCommand>(
CliConstants.DefaultCommandName, isDefaultCommand: true);
}
public static void ShouldBeBranch <TSettings>(this ConfiguredCommand command)
{
command.CommandType.ShouldBeNull();
command.SettingsType.ShouldBe <TSettings>();
}
internal static MaterializationDaemonConfig CreateMaterializationDaemonConfig(ConfiguredCommand conf)
{
return(new MaterializationDaemonConfig(
maxDegreeOfParallelism: conf.Get(MaxDegreeOfParallelism),
logDir: conf.Get(LogDir)));
}
public static void ShouldBeCommand <TCommand, TSettings>(this ConfiguredCommand command)
{
command.CommandType.ShouldBe <TCommand>();
command.SettingsType.ShouldBe <TSettings>();
}
private async Task <IIpcResult> MaterializeDirectoriesAsync(ConfiguredCommand conf)
{
m_counters.IncrementCounter(MaterializationDaemonCounter.MaterializeDirectoriesRequestCount);
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
var directoryFilterKinds = DirectoryContentFilterKind.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryFilters.Length || directoryPaths.Length != directoryFilterKinds.Length)
{
return(new IpcResult(
IpcResultStatus.InvalidInput,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #directoryFilters = {directoryFilters.Length}, #directoryFilterKinds = {directoryFilterKinds.Length}")));
}
if (ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var possibleFilterKinds = ParseFilterKinds(directoryFilterKinds);
if (!possibleFilterKinds.Succeeded)
{
return(new IpcResult(IpcResultStatus.InvalidInput, possibleFilterKinds.Failure.Describe()));
}
var initializedFilters = possibleFilters.Result;
var filterKinds = possibleFilterKinds.Result;
var possibleFiles = await GetUniqueFilteredDirectoryContentAsync(directoryPaths, directoryIds, initializedFilters, filterKinds);
if (!possibleFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFiles.Failure.Describe()));
}
var filesToMaterialize = possibleFiles.Result;
var sw = Stopwatch.StartNew();
using (m_counters.StartStopwatch(MaterializationDaemonCounter.MaterializeDirectoriesOuterMaterializationDuration))
{
try
{
await m_actionQueue.ForEachAsync(
filesToMaterialize,
async (file, i) =>
{
m_counters.AddToCounter(MaterializationDaemonCounter.MaterializeDirectoriesMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(file.Artifact, file.FileName, ignoreMaterializationFailures: false);
});
}
catch (Exception e)
{
return(new IpcResult(
IpcResultStatus.GenericError,
e.ToStringDemystified()));
}
}
m_counters.IncrementCounter(MaterializationDaemonCounter.MaterializeDirectoriesFilesToMaterialize);
return(IpcResult.Success(
$"Materialized files ({filesToMaterialize.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, filesToMaterialize.Select(f => f.FileName))}"));
}
/// <summary>
/// Takes a list of sealed directories, searches their contents for special manifest files, parses each of them to obtain
/// a list of files to materialize, and finally requests from BuildXL's ApiServer to materialize them.
/// </summary>
private async Task <IIpcResult> RegisterManifestInternalAsync(ConfiguredCommand conf)
{
m_counters.IncrementCounter(MaterializationDaemonCounter.RegisterManifestRequestCount);
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #directoryFilters = {directoryFilters.Length}")));
}
if (ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var initializedFilters = possibleFilters.Result;
var possibleManifestFiles = await GetUniqueFilteredDirectoryContentAsync(directoryPaths, directoryIds, initializedFilters);
if (!possibleManifestFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleManifestFiles.Failure.Describe()));
}
var manifestFiles = possibleManifestFiles.Result;
var sw = Stopwatch.StartNew();
List <string> filesToMaterialize = null;
try
{
// ensure that the manifest files are actually present on disk
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
manifestFiles,
async (manifest, i) =>
{
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(manifest.Artifact, manifest.FileName, ignoreMaterializationFailures: false);
});
}
Possible <List <string> > possibleFiles;
using (m_counters.StartStopwatch(MaterializationDaemonCounter.ManifestParsingOuterDuration))
{
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingTotalFiles, possibleManifestFiles.Result.Count);
possibleFiles = await ParseManifestFilesAsync(manifestFiles);
if (!possibleFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFiles.Failure.Describe()));
}
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingReferencedTotalFiles, possibleFiles.Result.Count);
}
filesToMaterialize = possibleFiles.Result;
sw = Stopwatch.StartNew();
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
filesToMaterialize,
async (file, i) =>
{
// Since these file paths are not real build artifacts (i.e., just strings constructed by a parser),
// they might not be "known" to BuildXL, and because of that, BuildXL won't be able to materialize them.
// Manifests are known to contain references to files that are not produced during a build, so we are
// ignoring materialization failures for such files. We are doing this so we won't fail IPC pips and
// a build could succeed.
// If there are real materialization errors (e.g., cache failure), the daemon relies on BuildXL logging
// the appropriate error events and failing the build.
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(FileArtifact.Invalid, file, ignoreMaterializationFailures: true);
});
}
}
catch (Exception e)
{
return(new IpcResult(
IpcResultStatus.GenericError,
e.ToStringDemystified()));
}
// Note: we are not claiming here that all the files in filesToMaterialize were materialized cause we are ignoring materialization failures.
// The real materialization status will be logged during execution of the Finalize command.
return(IpcResult.Success(
$"Processed paths ({filesToMaterialize.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, filesToMaterialize)}"));
}
public CommandConfigurator(ConfiguredCommand command)
{
Command = command;
}
