public NetworkCredential Write(string target, string username, string password) { Trace.Entering(); ArgUtil.NotNullOrEmpty(target, nameof(target)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); try { UnlockKeyChain(); // base64encode username + ':' + base64encode password // OSX keychain requires you provide -s target and -a username to retrieve password // So, we will trade both username and password as 'secret' store into keychain string usernameBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(username)); string passwordBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(password)); string secretForKeyChain = $"{usernameBase64}:{passwordBase64}"; List <string> securityOut = new List <string>(); List <string> securityError = new List <string>(); object outputLock = new object(); using (var p = HostContext.CreateService <IProcessInvoker>()) { p.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { lock (outputLock) { securityOut.Add(stdout.Data); } } }; p.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { lock (outputLock) { securityError.Add(stderr.Data); } } }; // make sure the 'security' has access to the key so we won't get prompt at runtime. int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), fileName: _securityUtil, arguments: $"add-generic-password -s {target} -a VSTSAGENT -w {secretForKeyChain} -T \"{_securityUtil}\" \"{_agentCredStoreKeyChain}\"", environment: null, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (exitCode == 0) { Trace.Info($"Successfully add-generic-password for {target} (VSTSAGENT)"); } else { if (securityOut.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityOut)); } if (securityError.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityError)); } throw new InvalidOperationException($"'security add-generic-password' failed with exit code {exitCode}."); } } return(new NetworkCredential(username, password)); } finally { LockKeyChain(); } }
private void LoadProxySetting() { string proxyConfigFile = HostContext.GetConfigFile(WellKnownConfigFile.Proxy); if (File.Exists(proxyConfigFile)) { // we expect the first line of the file is the proxy url Trace.Verbose($"Try read proxy setting from file: {proxyConfigFile}."); ProxyAddress = File.ReadLines(proxyConfigFile).FirstOrDefault() ?? string.Empty; ProxyAddress = ProxyAddress.Trim(); Trace.Verbose($"{ProxyAddress}"); } if (string.IsNullOrEmpty(ProxyAddress)) { Trace.Verbose("Try read proxy setting from environment variable: 'VSTS_HTTP_PROXY'."); ProxyAddress = Environment.GetEnvironmentVariable("VSTS_HTTP_PROXY") ?? string.Empty; ProxyAddress = ProxyAddress.Trim(); Trace.Verbose($"{ProxyAddress}"); } if (!string.IsNullOrEmpty(ProxyAddress) && !Uri.IsWellFormedUriString(ProxyAddress, UriKind.Absolute)) { Trace.Error($"The proxy url is not a well formed absolute uri string: {ProxyAddress}."); ProxyAddress = string.Empty; } if (!string.IsNullOrEmpty(ProxyAddress)) { Trace.Info($"Config proxy at: {ProxyAddress}."); string proxyCredFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyCredentials); if (File.Exists(proxyCredFile)) { string lookupKey = File.ReadAllLines(proxyCredFile).FirstOrDefault(); if (!string.IsNullOrEmpty(lookupKey)) { var credStore = HostContext.GetService <IAgentCredentialStore>(); var proxyCred = credStore.Read($"VSTS_AGENT_PROXY_{lookupKey}"); ProxyUsername = proxyCred.UserName; ProxyPassword = proxyCred.Password; } } if (string.IsNullOrEmpty(ProxyUsername)) { ProxyUsername = Environment.GetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME"); } if (string.IsNullOrEmpty(ProxyPassword)) { ProxyPassword = Environment.GetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD"); } if (!string.IsNullOrEmpty(ProxyPassword)) { HostContext.SecretMasker.AddValue(ProxyPassword); } if (string.IsNullOrEmpty(ProxyUsername) || string.IsNullOrEmpty(ProxyPassword)) { Trace.Info($"Config proxy use DefaultNetworkCredentials."); } else { Trace.Info($"Config authentication proxy as: {ProxyUsername}."); } string proxyBypassFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyBypass); if (File.Exists(proxyBypassFile)) { Trace.Verbose($"Try read proxy bypass list from file: {proxyBypassFile}."); foreach (string bypass in File.ReadAllLines(proxyBypassFile)) { if (string.IsNullOrWhiteSpace(bypass)) { continue; } else { Trace.Info($"Bypass proxy for: {bypass}."); ProxyBypassList.Add(bypass.Trim()); } } } _agentWebProxy.Update(ProxyAddress, ProxyUsername, ProxyPassword, ProxyBypassList); } else { Trace.Info($"No proxy setting found."); } }
public void Delete(string target) { Trace.Entering(); ArgUtil.NotNullOrEmpty(target, nameof(target)); try { UnlockKeyChain(); List <string> securityOut = new List <string>(); List <string> securityError = new List <string>(); object outputLock = new object(); using (var p = HostContext.CreateService <IProcessInvoker>()) { p.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { lock (outputLock) { securityOut.Add(stdout.Data); } } }; p.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { lock (outputLock) { securityError.Add(stderr.Data); } } }; int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), fileName: _securityUtil, arguments: $"delete-generic-password -s {target} -a VSTSAGENT \"{_agentCredStoreKeyChain}\"", environment: null, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (exitCode == 0) { Trace.Info($"Successfully delete-generic-password for {target} (VSTSAGENT)"); } else { if (securityOut.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityOut)); } if (securityError.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityError)); } throw new InvalidOperationException($"'security delete-generic-password' failed with exit code {exitCode}."); } } } finally { LockKeyChain(); } }
public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _securityUtil = WhichUtil.Which("security", true, Trace); _agentCredStoreKeyChain = hostContext.GetConfigFile(WellKnownConfigFile.CredentialStore); // Create osx key chain if it doesn't exists. if (!File.Exists(_agentCredStoreKeyChain)) { List <string> securityOut = new List <string>(); List <string> securityError = new List <string>(); object outputLock = new object(); using (var p = HostContext.CreateService <IProcessInvoker>()) { p.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { lock (outputLock) { securityOut.Add(stdout.Data); } } }; p.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { lock (outputLock) { securityError.Add(stderr.Data); } } }; // make sure the 'security' has access to the key so we won't get prompt at runtime. int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), fileName: _securityUtil, arguments: $"create-keychain -p {_osxAgentCredStoreKeyChainPassword} \"{_agentCredStoreKeyChain}\"", environment: null, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (exitCode == 0) { Trace.Info($"Successfully create-keychain for {_agentCredStoreKeyChain}"); } else { if (securityOut.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityOut)); } if (securityError.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityError)); } throw new InvalidOperationException($"'security create-keychain' failed with exit code {exitCode}."); } } } else { // Try unlock and lock the keychain, make sure it's still in good stage UnlockKeyChain(); LockKeyChain(); } }
public NetworkCredential Read(string target) { Trace.Entering(); ArgUtil.NotNullOrEmpty(target, nameof(target)); try { UnlockKeyChain(); string username; string password; List <string> securityOut = new List <string>(); List <string> securityError = new List <string>(); object outputLock = new object(); using (var p = HostContext.CreateService <IProcessInvoker>()) { p.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { lock (outputLock) { securityOut.Add(stdout.Data); } } }; p.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { lock (outputLock) { securityError.Add(stderr.Data); } } }; int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), fileName: _securityUtil, arguments: $"find-generic-password -s {target} -a VSTSAGENT -w -g \"{_agentCredStoreKeyChain}\"", environment: null, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (exitCode == 0) { string keyChainSecret = securityOut.First(); string[] secrets = keyChainSecret.Split(':'); if (secrets.Length == 2 && !string.IsNullOrEmpty(secrets[0]) && !string.IsNullOrEmpty(secrets[1])) { Trace.Info($"Successfully find-generic-password for {target} (VSTSAGENT)"); username = Encoding.UTF8.GetString(Convert.FromBase64String(secrets[0])); password = Encoding.UTF8.GetString(Convert.FromBase64String(secrets[1])); return(new NetworkCredential(username, password)); } else { throw new ArgumentOutOfRangeException(nameof(keyChainSecret)); } } else { if (securityOut.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityOut)); } if (securityError.Count > 0) { Trace.Error(string.Join(Environment.NewLine, securityError)); } throw new InvalidOperationException($"'security find-generic-password' failed with exit code {exitCode}."); } } } finally { LockKeyChain(); } }
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit) { Trace.Info("Stop aggressive process web console line queue."); _webConsoleLineAggressiveDequeue = false; } // Group consolelines by timeline record of each step Dictionary <Guid, List <TimelineRecordLogLine> > stepsConsoleLines = new Dictionary <Guid, List <TimelineRecordLogLine> >(); List <Guid> stepRecordIds = new List <Guid>(); // We need to keep lines in order int linesCounter = 0; ConsoleLineInfo lineInfo; while (_webConsoleLineQueue.TryDequeue(out lineInfo)) { if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId)) { stepsConsoleLines[lineInfo.StepRecordId] = new List <TimelineRecordLogLine>(); stepRecordIds.Add(lineInfo.StepRecordId); } if (!string.IsNullOrEmpty(lineInfo.Line) && lineInfo.Line.Length > 1024) { Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars"); lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}..."; } stepsConsoleLines[lineInfo.StepRecordId].Add(new TimelineRecordLogLine(lineInfo.Line, lineInfo.LineNumber)); linesCounter++; // process at most about 500 lines of web console line during regular timer dequeue task. if (!runOnce && linesCounter > 500) { break; } } // Batch post consolelines for each step timeline record foreach (var stepRecordId in stepRecordIds) { // Split consolelines into batch, each batch will container at most 100 lines. int batchCounter = 0; List <List <TimelineRecordLogLine> > batchedLines = new List <List <TimelineRecordLogLine> >(); foreach (var line in stepsConsoleLines[stepRecordId]) { var currentBatch = batchedLines.ElementAtOrDefault(batchCounter); if (currentBatch == null) { batchedLines.Add(new List <TimelineRecordLogLine>()); currentBatch = batchedLines.ElementAt(batchCounter); } currentBatch.Add(line); if (currentBatch.Count >= 100) { batchCounter++; } } if (batchedLines.Count > 0) { // When job finish, web console lines becomes less interesting to customer // We batch and produce 500 lines of web console output every 500ms // If customer's task produce massive of outputs, then the last queue drain run might take forever. // So we will only upload the last 200 lines of each step from all buffered web console lines. if (runOnce && batchedLines.Count > 2) { Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); batchedLines = batchedLines.TakeLast(2).ToList(); } int errorCount = 0; foreach (var batch in batchedLines) { try { // we will not requeue failed batch, since the web console lines are time sensitive. await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(x => x.Line).ToList(), batch[0].LineNumber, default(CancellationToken)); if (_firstConsoleOutputs) { HostContext.WritePerfCounter($"WorkerJobServerQueueAppendFirstConsoleOutput_{_planId.ToString()}"); _firstConsoleOutputs = false; } } catch (Exception ex) { Trace.Info("Catch exception during append web console line, keep going since the process is best effort."); Trace.Error(ex); errorCount++; } } Trace.Info("Try to append {0} batches web console lines for record '{2}', success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount, stepRecordId); } } if (runOnce) { break; } else { await Task.Delay(_webConsoleLineAggressiveDequeue?_aggressiveDelayForWebConsoleLineDequeue : _delayForWebConsoleLineDequeue); } } }
private void LoadProxySetting() { string proxyConfigFile = HostContext.GetConfigFile(WellKnownConfigFile.Proxy); if (File.Exists(proxyConfigFile)) { // we expect the first line of the file is the proxy url Trace.Verbose($"Try read proxy setting from file: {proxyConfigFile}."); ProxyAddress = File.ReadLines(proxyConfigFile).FirstOrDefault() ?? string.Empty; ProxyAddress = ProxyAddress.Trim(); Trace.Verbose($"{ProxyAddress}"); } if (string.IsNullOrEmpty(ProxyAddress)) { ProxyAddress = AgentKnobs.ProxyAddress.GetValue(HostContext).AsString(); Trace.Verbose($"Proxy address: {ProxyAddress}"); } if (!string.IsNullOrEmpty(ProxyAddress) && !Uri.IsWellFormedUriString(ProxyAddress, UriKind.Absolute)) { Trace.Error($"The proxy url is not a well formed absolute uri string: {ProxyAddress}."); ProxyAddress = string.Empty; } if (!string.IsNullOrEmpty(ProxyAddress)) { Trace.Info($"Config proxy at: {ProxyAddress}."); string proxyCredFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyCredentials); if (File.Exists(proxyCredFile)) { string lookupKey = File.ReadAllLines(proxyCredFile).FirstOrDefault(); if (!string.IsNullOrEmpty(lookupKey)) { var credStore = HostContext.GetService <IAgentCredentialStore>(); var proxyCred = credStore.Read($"VSTS_AGENT_PROXY_{lookupKey}"); ProxyUsername = proxyCred.UserName; ProxyPassword = proxyCred.Password; } } if (string.IsNullOrEmpty(ProxyUsername)) { ProxyUsername = AgentKnobs.ProxyUsername.GetValue(HostContext).AsString(); } if (string.IsNullOrEmpty(ProxyPassword)) { ProxyPassword = AgentKnobs.ProxyPassword.GetValue(HostContext).AsString(); } if (!string.IsNullOrEmpty(ProxyPassword)) { HostContext.SecretMasker.AddValue(ProxyPassword); } if (string.IsNullOrEmpty(ProxyUsername) || string.IsNullOrEmpty(ProxyPassword)) { Trace.Info($"Config proxy use DefaultNetworkCredentials."); } else { Trace.Info($"Config authentication proxy as: {ProxyUsername}."); } LoadProxyBypassList(); _agentWebProxy.Update(ProxyAddress, ProxyUsername, ProxyPassword, ProxyBypassList); } else { Trace.Info($"No proxy setting found."); } }