/// <summary> /// Creates a symbol request. /// </summary> public async Task <Request> CreateAsync(CancellationToken token) { if (!m_config.DomainId.HasValue) { m_logger.Verbose("DomainId is not specified. Creating symbol publishing request using DefaultDomainId."); } IDomainId domainId = m_config.DomainId.HasValue ? new ByteDomainId(m_config.DomainId.Value) : WellKnownDomainIds.DefaultDomainId; Request result; using (m_counters.StartStopwatch(SymbolClientCounter.CreateDuration)) { result = await m_symbolClient.CreateRequestAsync(domainId, RequestName, m_config.EnableChunkDedup, token); } m_requestId = result.Id; m_domainId = result.DomainId; // info about a request in a human-readable form var requestDetails = $"Symbol request has been created:{Environment.NewLine}" + $"ID: {result.Id}{Environment.NewLine}" + $"Name: {result.Name}{Environment.NewLine}" + $"Content list: '{result.Url}/DebugEntries'"; // Send the message to the main log. Analysis.IgnoreResult(await m_apiClient.LogMessage(requestDetails)); m_logger.Verbose(requestDetails); return(result); }
private async Task <IIpcResult> ParseAndExecuteCommandAsync(int id, IIpcOperation operation) { string cmdLine = operation.Payload; m_logger.Verbose($"Command received. Request #{id}, CommandLine: {cmdLine}"); ConfiguredCommand conf; using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration)) { conf = ParseArgs(cmdLine, m_parser); } IIpcResult result; using (var duration = m_counters.StartStopwatch(DaemonCounter.ServerActionDuration)) { result = await conf.Command.ServerAction(conf, this); result.ActionDuration = duration.Elapsed; } TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime; m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds); m_logger.Verbose($"Request #{id} processed in {queueDuration}, Result: {result.ExitCode}"); return(result); }
/// <summary> /// Executes and, if necessary, retries an operation /// </summary> protected async Task <U> RetryAsync <U>( string operationName, Func <T, CancellationToken, Task <U> > fn, CancellationToken cancellationToken, IEnumerator <TimeSpan> retryIntervalEnumerator = null, bool reloadFirst = false, Guid?operationId = null, TimeSpan?timeout = null) { operationId = operationId ?? Guid.NewGuid(); retryIntervalEnumerator = retryIntervalEnumerator ?? m_retryIntervals.GetEnumerator(); timeout = timeout ?? s_defaultOperationTimeout; try { using (CancellationTokenSource timeoutCancellationSource = new CancellationTokenSource()) using (CancellationTokenSource innerCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token)) { var instance = GetCurrentVersionedValue(); if (reloadFirst) { var reloaded = Reloader.Reload(instance.Version); m_logger.Warning("[{2}] Service client reloaded; new instance created: {0}, new client version: {1}", reloaded, Reloader.CurrentVersion, operationId.Value); } m_logger.Verbose("[{2}] Invoking '{0}' against instance version {1}", operationName, instance.Version, operationId.Value); return(await WithTimeoutAsync(fn(instance.Value, innerCancellationSource.Token), timeout.Value, timeoutCancellationSource)); } } catch (Exception e) when(m_nonRetryableExceptions.Contains(e.GetType())) { // We should not retry exceptions of this type. throw; } catch (Exception e) { if (e is TimeoutException) { m_logger.Warning("Timeout ({0}sec) happened while waiting {1}.", timeout.Value.TotalSeconds, operationName); } if (retryIntervalEnumerator.MoveNext()) { m_logger.Warning("[{2}] Waiting {1} before retrying on exception: {0}", e.ToString(), retryIntervalEnumerator.Current, operationId.Value); await Task.Delay(retryIntervalEnumerator.Current); return(await RetryAsync(operationName, fn, cancellationToken, retryIntervalEnumerator, reloadFirst : true, operationId : operationId)); } else { m_logger.Error("[{1}] Failing because number of retries were exhausted. Final exception: {0};", e.ToString(), operationId.Value); throw; } } }
/// <summary> /// Starts to listen for client connections. As soon as a connection is received, /// it is placed in an action block from which it is picked up and handled asynchronously /// (in the <see cref="ParseAndExecuteCommandAsync"/> method). /// </summary> public void Start() { m_server.Start(this); // some tests run without API Client, so there is no one to notify that the service is ready if (ApiClient != null) { var process = Process.GetCurrentProcess(); m_logger.Verbose($"Reporting to BuildXL that the service is ready (pid: {process.Id}, processName: '{process.ProcessName}')"); var possibleResult = ApiClient.ReportServicePipIsReady(process.Id, process.ProcessName).GetAwaiter().GetResult(); if (!possibleResult.Succeeded) { m_logger.Error("Failed to notify BuildXL that the service is ready."); } else { m_logger.Verbose("Successfully notified BuildXL that the service is ready."); } } }
private TcpIpConnectivity CreateAndStartTcpIpProvider(IIpcLogger logger, int numTimesToRetry = 3, int delayMillis = 250) { try { var port = Utils.GetUnusedPortNumber(); var tcpIpProvider = new TcpIpConnectivity(port); tcpIpProvider.StartListening(); return(tcpIpProvider); } catch (SocketException e) { if (numTimesToRetry > 0) { logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Waiting {delayMillis}ms then retrying {numTimesToRetry} more times"); Thread.Sleep(millisecondsTimeout: delayMillis); return(CreateAndStartTcpIpProvider(logger, numTimesToRetry - 1, delayMillis * 2)); } else { logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Not retrying any more."); throw; } } }
/// <summary> /// Parses a list of arguments and returns a ConfiguredCommand. /// </summary> public static ConfiguredCommand ParseArgs(string[] args, IParser parser, IIpcLogger logger = null, bool ignoreInvalidOptions = false) { var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage()); if (args.Length == 0) { throw new ArgumentException(I($"Command is required. {usageMessage.Value}")); } var argsQueue = new Queue <string>(args.Length); foreach (var arg in args) { if (arg[0] == ResponseFilePrefix) { foreach (var argFromFile in ProcessResponseFile(arg, parser)) { argsQueue.Enqueue(argFromFile); } } else { argsQueue.Enqueue(arg); } } string cmdName = argsQueue.Dequeue(); if (!Commands.TryGetValue(cmdName, out Command cmd)) { throw new ArgumentException(I($"No command '{cmdName}' is found. {usageMessage.Value}")); } var sw = Stopwatch.StartNew(); Config conf = BuildXL.Utilities.CLI.Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions); var parseTime = sw.Elapsed; logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), ServicePipDaemon.LogPrefix); logger.Verbose("Parsing command line arguments done in {0}", parseTime); return(new ConfiguredCommand(cmd, conf, logger)); }
/// <summary> /// Takes a hash as string and registers its corresponding SHA-256 ContentHash using BuildXL Api. /// Should be called only when DropConfig.GenerateSignedManifest is true. /// Returns a hashset of failing RelativePaths. /// </summary> private async Task <HashSet <string> > RegisterFilesForBuildManifestAsync(BuildManifestEntry[] buildManifestEntries) { await Task.Yield(); Contract.Requires(m_dropDaemon.DropConfig.GenerateSignedManifest == true, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig"); var bxlResult = await m_dropDaemon.ApiClient.RegisterFilesForBuildManifest(m_dropDaemon.DropName, buildManifestEntries); if (!bxlResult.Succeeded) { m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest unsuccessful. Failure: {bxlResult.Failure.DescribeIncludingInnerFailures()}"); return(new HashSet <string>(buildManifestEntries.Select(bme => bme.RelativePath))); } if (bxlResult.Result.Length > 0) { m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest found {bxlResult.Result.Length} file hashing failures."); return(new HashSet <string>(bxlResult.Result.Select(bme => bme.RelativePath))); } return(new HashSet <string>()); }
private VssCredentials GetCredentials() => new VsoCredentialHelper(m => m_logger.Verbose(m)) .GetCredentials(m_config.Service, true, null, null, PromptBehavior.Never);
private void Verbose(string msg, params object[] args) { m_logger.Verbose("(" + m_name + ") " + msg, args); }
/// <nodoc /> public VsoClient(IIpcLogger logger, Client bxlApiClient, DaemonConfig daemonConfig, DropConfig dropConfig) { Contract.Requires(daemonConfig != null); Contract.Requires(dropConfig != null); m_logger = logger; m_config = dropConfig; m_bxlApiClient = bxlApiClient; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m)); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); m_nagleQueue = NagleQueue <AddFileItem> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessAddFilesAsync); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(daemonConfig.LogDir, m_config.ArtifactLogName)); } }
/// <nodoc /> public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient) { m_logger = logger; m_apiClient = apiClient; m_config = config; m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior; m_cancellationSource = new CancellationTokenSource(); m_counters = new CounterCollection <SymbolClientCounter>(); m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}")); m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m)); m_symbolClient = new ReloadingSymbolClient( logger: logger, clientConstructor: CreateSymbolServiceClient); m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessBatchedFilesAsync); m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads); }