public async Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation(@"Startup Leader Elector for operator ""{operatorName}"".", _settings.Name); _leaseCheck?.Dispose(); _leaseCheck = new Timer( TimeSpan.FromSeconds(_settings.LeaderElectionCheckInterval).TotalMilliseconds) { AutoReset = true, }; _logger.LogTrace("Fetching namespace for leader election."); _namespace = await _client.GetCurrentNamespace(); _operatorDeployment = (await _client.List <V1Deployment>( _namespace, new EqualsSelector("operator-deployment", _settings.Name))).FirstOrDefault(); if (_operatorDeployment != null) { _operatorDeployment.Kind = V1Deployment.KubeKind; _operatorDeployment.ApiVersion = $"{V1Deployment.KubeGroup}/{V1Deployment.KubeApiVersion}"; } #if DEBUG _election.LeadershipChanged(LeaderState.Leader); #else _leaseCheck.Start(); _leaseCheck.Elapsed += async(_, __) => await CheckLeaderLease(); await CheckLeaderLease(); #endif }
/// <summary> /// Check the <see cref="V1Lease"/> object for the leader election. /// </summary> /// <returns>A Task.</returns> internal async Task CheckLeaderLease() { if (_namespace.Length == 0) { _logger.LogTrace("Fetching namespace for leader election."); _namespace = await _client.GetCurrentNamespace(); } _logger.LogTrace(@"Fetch V1Lease object for operator ""{operator}"".", _settings.Name); var lease = await _client.Get <V1Lease>(_leaseName, _namespace); // If the lease does not exist, create it, set this instance as leader // fire the appropriate event, return. if (lease == null) { _logger.LogInformation( @"There was no lease for operator ""{operator}"". Creating one and electing ""{hostname}"" as leader.", _settings.Name, _hostname); try { await _client.Create( new V1Lease( $"{V1Lease.KubeGroup}/{V1Lease.KubeApiVersion}", V1Lease.KubeKind, new V1ObjectMeta( name: _leaseName, namespaceProperty: _namespace, annotations: new Dictionary <string, string> { { "leader-elector", _settings.Name } }), new V1LeaseSpec( DateTime.UtcNow, _hostname, _settings.LeaderElectionLeaseDuration, 0, DateTime.UtcNow))); _election.LeadershipChanged(LeaderState.Leader); } catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict) { _logger.LogInformation("Another instance of the operator was faster. Falling back to candiate."); _election.LeadershipChanged(LeaderState.Candidate); } catch (HttpOperationException ex) { _logger.LogCritical( ex, @"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""", _hostname, _settings.Name, $"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}"); } catch (Exception ex) { _logger.LogCritical( ex, @"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".", _hostname, _settings.Name); } return; } /* * If the lease exists, check if this instance is the leader. * If it is, update the renew time, and update the entity. * If it isn't and the lease time is in the past, * set the leader, update the entity, trigger event. */ if (lease.Spec.HolderIdentity == _hostname) { _logger.LogDebug( @"The instance ""{hostname}"" is still the leader for operator ""{operator}"".", _hostname, _settings.Name); lease.Spec.RenewTime = DateTime.UtcNow; try { await _client.Update(lease); _election.LeadershipChanged(LeaderState.Leader); } catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict) { _logger.LogWarning("Another instance updated the lease. Retry on next cycle."); } catch (HttpOperationException ex) { _logger.LogCritical( ex, @"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""", _hostname, _settings.Name, $"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}"); } catch (Exception ex) { _logger.LogCritical( ex, @"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".", _hostname, _settings.Name); } return; } if (lease.Spec.RenewTime.HasValue && lease.Spec.RenewTime.Value + TimeSpan.FromSeconds(lease.Spec.LeaseDurationSeconds ?? _settings.LeaderElectionLeaseDuration) < DateTime.UtcNow) { _logger.LogInformation( @"The lease for operator ""{operator}"" ran out. Electing ""{hostname}"" as leader.", _settings.Name, _hostname); lease.Spec.AcquireTime = DateTime.UtcNow; lease.Spec.RenewTime = DateTime.UtcNow; lease.Spec.HolderIdentity = _hostname; lease.Spec.LeaseTransitions ??= 0; lease.Spec.LeaseTransitions += 1; try { await _client.Update(lease); _election.LeadershipChanged(LeaderState.Leader); } catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict) { _logger.LogWarning("Another instance updated the lease. Retry on next cycle."); } catch (HttpOperationException ex) { _logger.LogCritical( ex, @"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""", _hostname, _settings.Name, $"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}"); } catch (Exception ex) { _logger.LogCritical( ex, @"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".", _hostname, _settings.Name); } return; } _logger.LogDebug( @"The lease for operator ""{operator}"" did not ran out, staying/becoming candidate.", _settings.Name); _election.LeadershipChanged(LeaderState.Candidate); }
public async Task Should_Return_Namespace() { var ns = await _client.GetCurrentNamespace(); ns.Should().Be("default"); }
public async Task <int> OnExecuteAsync(CommandLineApplication app) { var @namespace = await _client.GetCurrentNamespace(); using var certManager = new CertificateGenerator(app.Out); #if DEBUG CertificatesPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); CaCertificatesPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); await certManager.CreateCaCertificateAsync(CaCertificatesPath); #endif Directory.CreateDirectory(CertificatesPath); File.Copy(Path.Join(CaCertificatesPath, "ca.pem"), Path.Join(Path.Join(CertificatesPath, "ca.pem"))); await certManager.CreateServerCertificateAsync( CertificatesPath, _settings.Name, @namespace, Path.Join(CaCertificatesPath, "ca.pem"), Path.Join(CaCertificatesPath, "ca-key.pem")); var deployment = (await _client.List <V1Deployment>( @namespace, new EqualsSelector("operator-deployment", _settings.Name))).FirstOrDefault(); if (deployment != null) { deployment.Kind = V1Deployment.KubeKind; deployment.ApiVersion = $"{V1Deployment.KubeGroup}/{V1Deployment.KubeApiVersion}"; } await app.Out.WriteLineAsync("Create service."); await _client.Delete <V1Service>(_settings.Name, @namespace); await _client.Create( new V1Service( V1Service.KubeApiVersion, V1Service.KubeKind, new V1ObjectMeta( name: _settings.Name, namespaceProperty: @namespace, ownerReferences: deployment != null ? new List <V1OwnerReference> { deployment.MakeOwnerReference(), } : null, labels: new Dictionary <string, string> { { "operator", _settings.Name }, { "usage", "webhook-service" }, }), new V1ServiceSpec { Ports = new List <V1ServicePort> { new() { Name = "https", TargetPort = "https", Port = 443, }, },