/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
m_nagleQueue = NagleQueue <AddFileItem> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessAddFilesAsync);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
/// <nodoc/>
public GenericServer([CanBeNull] string name, IServerConfig config, ListenerSourceBlock <TClient> .CancellableListener listener, bool clientFailuresAreFatal = false)
{
Contract.Requires(config != null);
Config = config;
Logger = config.Logger ?? VoidLogger.Instance;
Name = name ?? "GenericServer";
m_diagnostics = new ConcurrentQueue <Exception>();
// in an event loop, wait for clients to connect via the 'AcceptClient' method
m_clientListenerBlock = new ListenerSourceBlock <TClient>(listener, name + ".EventLoop", Logger);
// handle each connected client via the 'ClientHandler' method
var clientHandlerAction = clientFailuresAreFatal
? (Func <TClient, Task>)ClientHandlerWithoutExceptionHandling
: (Func <TClient, Task>)ClientHandlerWithExceptionHandling;
var clientHandlerBlock = new ActionBlock <TClient>(
clientHandlerAction,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = config.MaxConcurrentClients,
});
// link the event loop to feed the clientHandler ActionBlock
m_clientListenerBlock.LinkTo(clientHandlerBlock, propagateCompletion: true);
}
/// <nodoc/>
public ListenerSourceBlock(CancellableListener listener, string name = "unknown", IIpcLogger logger = null)
{
Contract.Requires(listener != null);
m_logger = logger ?? VoidLogger.Instance;
m_name = name ?? "ListenerSourceBlock";
m_listener = listener;
m_stopTask = TaskSourceSlim.Create <Unit>();
m_startCounter = 0;
}
/// <nodoc />
public ReloadingClient(IIpcLogger logger, Func <T> clientConstructor, IEnumerable <TimeSpan> retryIntervals = null, IEnumerable <Type> nonRetryableExceptions = null)
{
Contract.Assert(logger != null);
Contract.Assert(clientConstructor != null);
Contract.Assert(nonRetryableExceptions == null || nonRetryableExceptions.All(e => e.IsSubclassOf(typeof(Exception))));
m_logger = logger;
Reloader = new Reloader <T>(clientConstructor, destructor: client => client.Dispose());
m_retryIntervals = retryIntervals ?? s_defaultRetryIntervals;
m_nonRetryableExceptions = nonRetryableExceptions == null ? new HashSet <Type>() : new HashSet <Type>(nonRetryableExceptions);
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = CreateSymbolServiceClient();
}
/// <nodoc />
public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, IIpcLogger logger, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Config = daemonConfig;
m_parser = parser;
ApiClient = client;
m_logger = logger;
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
}
// ==================================================================================================
// Constructor
// ==================================================================================================
/// <nodoc/>
public DaemonConfig(
IIpcLogger logger,
string moniker,
int?maxConnectRetries = null,
TimeSpan?connectRetryDelay = null,
bool?stopOnFirstFailure = null,
bool?enableCloudBuildIntegration = null)
{
Contract.Requires(logger != null);
Moniker = moniker;
Logger = logger;
MaxConnectRetries = maxConnectRetries ?? DefaultMaxConnectRetries;
ConnectRetryDelay = connectRetryDelay ?? DefaultConnectRetryDelay;
StopOnFirstFailure = stopOnFirstFailure ?? DefaultStopOnFirstFailure;
EnableCloudBuildIntegration = enableCloudBuildIntegration ?? DefaultEnableCloudBuildIntegration;
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
}
/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
// create dataflow blocks
var groupingOptions = new GroupingDataflowBlockOptions {
Greedy = true
};
var actionOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = m_config.MaxParallelUploads
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions);
m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling
m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions);
m_batchBlock.LinkTo(m_bufferBlock, linkOptions);
m_bufferBlock.LinkTo(m_actionBlock, linkOptions);
// create and set up timer for triggering the batch block
TimeSpan timerInterval = m_config.NagleTime;
m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
/// <summary>
/// Parses a list of arguments and returns a ConfiguredCommand.
/// </summary>
public static ConfiguredCommand ParseArgs(string[] args, IParser parser, IIpcLogger logger = null, bool ignoreInvalidOptions = false)
{
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
if (args.Length == 0)
{
throw new ArgumentException(I($"Command is required. {usageMessage.Value}"));
}
var argsQueue = new Queue <string>(args.Length);
foreach (var arg in args)
{
if (arg[0] == ResponseFilePrefix)
{
foreach (var argFromFile in ProcessResponseFile(arg, parser))
{
argsQueue.Enqueue(argFromFile);
}
}
else
{
argsQueue.Enqueue(arg);
}
}
string cmdName = argsQueue.Dequeue();
if (!Commands.TryGetValue(cmdName, out Command cmd))
{
throw new ArgumentException(I($"No command '{cmdName}' is found. {usageMessage.Value}"));
}
var sw = Stopwatch.StartNew();
Config conf = BuildXL.Utilities.CLI.Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
var parseTime = sw.Elapsed;
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), ServicePipDaemon.LogPrefix);
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
return(new ConfiguredCommand(cmd, conf, logger));
}
/// <nodoc/>
public MultiplexingServer([CanBeNull] string name, [CanBeNull] IIpcLogger logger, IConnectivityProvider <TClient> connectivityProvider, int maxConcurrentClients, int maxConcurrentRequestsPerClient)
{
Contract.Requires(connectivityProvider != null);
Contract.Requires(maxConcurrentClients > 0);
Contract.Requires(maxConcurrentRequestsPerClient > 0);
Name = name ?? GetType().Name;
Logger = logger ?? VoidLogger.Instance;
m_connectivityProvider = connectivityProvider;
m_clientHandlingConfig = new ServerConfig {
Logger = Logger, MaxConcurrentClients = maxConcurrentClients
};
m_requestHandlingConfig = new ServerConfig {
Logger = Logger, MaxConcurrentClients = maxConcurrentRequestsPerClient
};
m_clientListener = new GenericServer <TClient>(
name: Name + ".ClientHandler",
config: m_clientHandlingConfig,
listener: (token) => connectivityProvider.AcceptClientAsync(token));
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_batchBlock = new TimedBatchBlock <BatchedSymbolFile>(
maxDegreeOfParallelism: m_config.MaxParallelUploads,
batchSize: m_config.BatchSize,
nagleInterval: m_config.NagleTime,
batchProcessor: ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
private TcpIpConnectivity CreateAndStartTcpIpProvider(IIpcLogger logger, int numTimesToRetry = 3, int delayMillis = 250)
{
try
{
var port = Utils.GetUnusedPortNumber();
var tcpIpProvider = new TcpIpConnectivity(port);
tcpIpProvider.StartListening();
return(tcpIpProvider);
}
catch (SocketException e)
{
if (numTimesToRetry > 0)
{
logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Waiting {delayMillis}ms then retrying {numTimesToRetry} more times");
Thread.Sleep(millisecondsTimeout: delayMillis);
return(CreateAndStartTcpIpProvider(logger, numTimesToRetry - 1, delayMillis * 2));
}
else
{
logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Not retrying any more.");
throw;
}
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="clientConstructor">Target drop service client.</param>
/// <param name="retryIntervals">How many times to retry and how much to wait between retries.</param>
public ReloadingDropServiceClient(IIpcLogger logger, Func<IDropServiceClient> clientConstructor, IEnumerable<TimeSpan> retryIntervals = null)
: base(logger, clientConstructor, retryIntervals)
{
}
/// <nodoc/>
public BuildXLBasedCloudBuildLogger(IIpcLogger logger, bool enableCloudBuildIntegration)
{
m_localLogger = logger;
m_etwEventSource = enableCloudBuildIntegration ? CloudBuildEventSource.Log : CloudBuildEventSource.TestLog;
}
/// <nodoc /> public static void Verbose(this IIpcLogger logger, string format, params object[] args) => logger?.Log(LogLevel.Verbose, format, args);
/// <summary>
/// Parses a string and returns a ConfiguredCommand.
/// </summary>
public static ConfiguredCommand ParseArgs(string allArgs, IParser parser, IIpcLogger logger = null, bool ignoreInvalidOptions = false)
{
return(ParseArgs(parser.SplitArgs(allArgs), parser, logger, ignoreInvalidOptions));
}
/// <nodoc /> public static void Warning(this IIpcLogger logger, string format, params object[] args) => logger?.Log(LogLevel.Warning, format, args);
/// <nodoc />
public IpcProviderWithMemoization(IIpcProvider provider, IIpcLogger defaultClientLogger = null)
{
m_provider = provider;
m_defaultClientLogger = defaultClientLogger;
}
/// <nodoc/>
public ReloadingSymbolClient(IIpcLogger logger, Func <ISymbolServiceClient> clientConstructor, IEnumerable <TimeSpan> retryIntervals = null)
: base(logger, clientConstructor, retryIntervals, new[] { typeof(DebugEntryExistsException) })
{
}
/// <nodoc/>
public ConfiguredCommand(Command command, Config config, IIpcLogger logger)
{
Command = command;
Config = config;
Logger = logger;
}
/// <nodoc /> public static void Error(this IIpcLogger logger, string format, params object[] args) => logger?.Log(LogLevel.Error, format, args);
/// <summary>
/// Constructor.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="clientConstructor">Target drop service client.</param>
/// <param name="retryIntervals">How many times to retry and how much to wait between retries.</param>
public ReloadingDropServiceClient(IIpcLogger logger, Func <IDropServiceClient> clientConstructor, IEnumerable <TimeSpan> retryIntervals = null)
{
m_logger = logger;
m_reloader = new Reloader <IDropServiceClient>(clientConstructor, destructor: client => client.Dispose());
m_retryIntervals = retryIntervals ?? s_defaultRetryIntervals;
}
/// <inheritdoc />
public FinalizedByCreatorServicePipDaemon(IParser parser, DaemonConfig daemonConfig, IIpcLogger logger, IIpcProvider rpcProvider = null, Client client = null)
: base(parser, daemonConfig, logger, rpcProvider, client)
{
}
/// <nodoc />
public SBOMLoggingWrapper(IIpcLogger logger)
{
m_innerLogger = logger;
}
