/// <summary>
/// Creates a symbol request.
/// </summary>
public async Task <Request> CreateAsync(CancellationToken token)
{
if (!m_config.DomainId.HasValue)
{
m_logger.Verbose("DomainId is not specified. Creating symbol publishing request using DefaultDomainId.");
}
IDomainId domainId = m_config.DomainId.HasValue
? new ByteDomainId(m_config.DomainId.Value)
: WellKnownDomainIds.DefaultDomainId;
Request result;
using (m_counters.StartStopwatch(SymbolClientCounter.CreateDuration))
{
result = await m_symbolClient.CreateRequestAsync(domainId, RequestName, m_config.EnableChunkDedup, token);
}
m_requestId = result.Id;
m_domainId = result.DomainId;
// info about a request in a human-readable form
var requestDetails = $"Symbol request has been created:{Environment.NewLine}"
+ $"ID: {result.Id}{Environment.NewLine}"
+ $"Name: {result.Name}{Environment.NewLine}"
+ $"Content list: '{result.Url}/DebugEntries'";
// Send the message to the main log.
Analysis.IgnoreResult(await m_apiClient.LogMessage(requestDetails));
m_logger.Verbose(requestDetails);
return(result);
}
private async Task <IIpcResult> ParseAndExecuteCommandAsync(int id, IIpcOperation operation)
{
string cmdLine = operation.Payload;
m_logger.Verbose($"Command received. Request #{id}, CommandLine: {cmdLine}");
ConfiguredCommand conf;
using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration))
{
conf = ParseArgs(cmdLine, m_parser);
}
IIpcResult result;
using (var duration = m_counters.StartStopwatch(DaemonCounter.ServerActionDuration))
{
result = await conf.Command.ServerAction(conf, this);
result.ActionDuration = duration.Elapsed;
}
TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime;
m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds);
m_logger.Verbose($"Request #{id} processed in {queueDuration}, Result: {result.ExitCode}");
return(result);
}
/// <summary>
/// Executes and, if necessary, retries an operation
/// </summary>
protected async Task <U> RetryAsync <U>(
string operationName,
Func <T, CancellationToken, Task <U> > fn,
CancellationToken cancellationToken,
IEnumerator <TimeSpan> retryIntervalEnumerator = null,
bool reloadFirst = false,
Guid?operationId = null,
TimeSpan?timeout = null)
{
operationId = operationId ?? Guid.NewGuid();
retryIntervalEnumerator = retryIntervalEnumerator ?? m_retryIntervals.GetEnumerator();
timeout = timeout ?? s_defaultOperationTimeout;
try
{
using (CancellationTokenSource timeoutCancellationSource = new CancellationTokenSource())
using (CancellationTokenSource innerCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token))
{
var instance = GetCurrentVersionedValue();
if (reloadFirst)
{
var reloaded = Reloader.Reload(instance.Version);
m_logger.Warning("[{2}] Service client reloaded; new instance created: {0}, new client version: {1}", reloaded, Reloader.CurrentVersion, operationId.Value);
}
m_logger.Verbose("[{2}] Invoking '{0}' against instance version {1}", operationName, instance.Version, operationId.Value);
return(await WithTimeoutAsync(fn(instance.Value, innerCancellationSource.Token), timeout.Value, timeoutCancellationSource));
}
}
catch (Exception e) when(m_nonRetryableExceptions.Contains(e.GetType()))
{
// We should not retry exceptions of this type.
throw;
}
catch (Exception e)
{
if (e is TimeoutException)
{
m_logger.Warning("Timeout ({0}sec) happened while waiting {1}.", timeout.Value.TotalSeconds, operationName);
}
if (retryIntervalEnumerator.MoveNext())
{
m_logger.Warning("[{2}] Waiting {1} before retrying on exception: {0}", e.ToString(), retryIntervalEnumerator.Current, operationId.Value);
await Task.Delay(retryIntervalEnumerator.Current);
return(await RetryAsync(operationName, fn, cancellationToken, retryIntervalEnumerator, reloadFirst : true, operationId : operationId));
}
else
{
m_logger.Error("[{1}] Failing because number of retries were exhausted. Final exception: {0};", e.ToString(), operationId.Value);
throw;
}
}
}
/// <summary>
/// Starts to listen for client connections. As soon as a connection is received,
/// it is placed in an action block from which it is picked up and handled asynchronously
/// (in the <see cref="ParseAndExecuteCommandAsync"/> method).
/// </summary>
public void Start()
{
m_server.Start(this);
// some tests run without API Client, so there is no one to notify that the service is ready
if (ApiClient != null)
{
var process = Process.GetCurrentProcess();
m_logger.Verbose($"Reporting to BuildXL that the service is ready (pid: {process.Id}, processName: '{process.ProcessName}')");
var possibleResult = ApiClient.ReportServicePipIsReady(process.Id, process.ProcessName).GetAwaiter().GetResult();
if (!possibleResult.Succeeded)
{
m_logger.Error("Failed to notify BuildXL that the service is ready.");
}
else
{
m_logger.Verbose("Successfully notified BuildXL that the service is ready.");
}
}
}
private TcpIpConnectivity CreateAndStartTcpIpProvider(IIpcLogger logger, int numTimesToRetry = 3, int delayMillis = 250)
{
try
{
var port = Utils.GetUnusedPortNumber();
var tcpIpProvider = new TcpIpConnectivity(port);
tcpIpProvider.StartListening();
return(tcpIpProvider);
}
catch (SocketException e)
{
if (numTimesToRetry > 0)
{
logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Waiting {delayMillis}ms then retrying {numTimesToRetry} more times");
Thread.Sleep(millisecondsTimeout: delayMillis);
return(CreateAndStartTcpIpProvider(logger, numTimesToRetry - 1, delayMillis * 2));
}
else
{
logger.Verbose($"Could not connect; error: '{e.GetLogEventMessage()}'. Not retrying any more.");
throw;
}
}
}
/// <summary>
/// Parses a list of arguments and returns a ConfiguredCommand.
/// </summary>
public static ConfiguredCommand ParseArgs(string[] args, IParser parser, IIpcLogger logger = null, bool ignoreInvalidOptions = false)
{
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
if (args.Length == 0)
{
throw new ArgumentException(I($"Command is required. {usageMessage.Value}"));
}
var argsQueue = new Queue <string>(args.Length);
foreach (var arg in args)
{
if (arg[0] == ResponseFilePrefix)
{
foreach (var argFromFile in ProcessResponseFile(arg, parser))
{
argsQueue.Enqueue(argFromFile);
}
}
else
{
argsQueue.Enqueue(arg);
}
}
string cmdName = argsQueue.Dequeue();
if (!Commands.TryGetValue(cmdName, out Command cmd))
{
throw new ArgumentException(I($"No command '{cmdName}' is found. {usageMessage.Value}"));
}
var sw = Stopwatch.StartNew();
Config conf = BuildXL.Utilities.CLI.Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
var parseTime = sw.Elapsed;
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), ServicePipDaemon.LogPrefix);
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
return(new ConfiguredCommand(cmd, conf, logger));
}
/// <summary>
/// Takes a hash as string and registers its corresponding SHA-256 ContentHash using BuildXL Api.
/// Should be called only when DropConfig.GenerateSignedManifest is true.
/// Returns a hashset of failing RelativePaths.
/// </summary>
private async Task <HashSet <string> > RegisterFilesForBuildManifestAsync(BuildManifestEntry[] buildManifestEntries)
{
await Task.Yield();
Contract.Requires(m_dropDaemon.DropConfig.GenerateSignedManifest == true, "RegisterFileForBuildManifest API called even though Build Manifest Generation is Disabled in DropConfig");
var bxlResult = await m_dropDaemon.ApiClient.RegisterFilesForBuildManifest(m_dropDaemon.DropName, buildManifestEntries);
if (!bxlResult.Succeeded)
{
m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest unsuccessful. Failure: {bxlResult.Failure.DescribeIncludingInnerFailures()}");
return(new HashSet <string>(buildManifestEntries.Select(bme => bme.RelativePath)));
}
if (bxlResult.Result.Length > 0)
{
m_logger.Verbose($"ApiClient.RegisterFileForBuildManifest found {bxlResult.Result.Length} file hashing failures.");
return(new HashSet <string>(bxlResult.Result.Select(bme => bme.RelativePath)));
}
return(new HashSet <string>());
}
private VssCredentials GetCredentials() => new VsoCredentialHelper(m => m_logger.Verbose(m)) .GetCredentials(m_config.Service, true, null, null, PromptBehavior.Never);
private void Verbose(string msg, params object[] args)
{
m_logger.Verbose("(" + m_name + ") " + msg, args);
}
/// <nodoc />
public VsoClient(IIpcLogger logger, Client bxlApiClient, DaemonConfig daemonConfig, DropConfig dropConfig)
{
Contract.Requires(daemonConfig != null);
Contract.Requires(dropConfig != null);
m_logger = logger;
m_config = dropConfig;
m_bxlApiClient = bxlApiClient;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m));
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
m_nagleQueue = NagleQueue <AddFileItem> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessAddFilesAsync);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(daemonConfig.LogDir, m_config.ArtifactLogName));
}
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
