/// <summary> /// Attempts to reconcile a child and parent object. /// </summary> /// <param name="parent"> /// The parent object being reconciled. /// </param> /// <param name="child"> /// The child object being reconciled, or <see langword="null"/> if a child object does not yet exist. /// </param> /// <param name="cancellationToken"> /// A <see cref="CancellationToken"/> which can be used to cancel the asynchronous operation. /// </param> /// <returns> /// A <see cref="Task"/> representing the asynchronous operation. /// </returns> public async Task ReconcileAsync(TParent parent, TChild child, CancellationToken cancellationToken) { // Let's assume Kubernetes takes care of garbage collection, so the source // object is always present. // // Although children with no parents are possible because of cascading background deletions: // https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#background-cascading-deletion, // they should never enter the reconcile loop. Debug.Assert(parent != null, "Cannot have a child object without a parent object"); if (!await this.reconciliationSemaphore.WaitAsync(0).ConfigureAwait(false)) { throw new InvalidOperationException("Only one instance of ReconcileAsync can run at a time."); } try { this.logger.LogInformation( "Scheduling reconciliation for parent {parent} and child {child} for operator {operatorName}", parent?.Metadata?.Name, child?.Metadata?.Name, this.configuration.OperatorName); if (child == null) { // Create a new object child = new TChild() { Metadata = new V1ObjectMeta() { Labels = new Dictionary <string, string>(this.configuration.ChildLabels), Name = parent.Metadata.Name, NamespaceProperty = parent.Metadata.NamespaceProperty, OwnerReferences = new V1OwnerReference[] { parent.AsOwnerReference(blockOwnerDeletion: false, controller: false), }, }, }; child.SetLabel(Annotations.ManagedBy, this.configuration.OperatorName); this.childFactory(parent, child); await this.childClient.CreateAsync(child, cancellationToken).ConfigureAwait(false); } else { this.logger.LogInformation( "Running {feedbackCount} feedback loops for parent {parent} and child {child} for operator {operatorName} because child is not null.", this.feedbackLoops.Count, parent?.Metadata?.Name, child?.Metadata?.Name, this.configuration.OperatorName); Feedback <TParent, TChild> feedback = null; using (var scope = this.services.CreateScope()) { var context = new ChildOperatorContext <TParent, TChild>( parent, child, scope.ServiceProvider); foreach (var feedbackLoop in this.feedbackLoops) { if ((feedback = await feedbackLoop(context, cancellationToken).ConfigureAwait(false)) != null) { if (feedback.ParentFeedback != null) { this.logger.LogInformation( "Applying patch {feedback} to parent {parent} for operator {operatorName}.", feedback, context.Parent?.Metadata?.Name, this.configuration.OperatorName); await this.parentClient.PatchAsync(context.Parent, feedback.ParentFeedback, cancellationToken).ConfigureAwait(false); } if (feedback.ChildFeedback != null) { this.logger.LogInformation( "Applying patch {feedback} to child {child} for operator {operatorName}.", feedback, context.Child?.Metadata?.Name, this.configuration.OperatorName); await this.childClient.PatchAsync(context.Child, feedback.ChildFeedback, cancellationToken).ConfigureAwait(false); } } } } } } catch (Exception ex) { this.logger.LogError(ex, "Caught error {errorMessage} while executing reconciliation for operator {operator}", ex.Message, this.configuration.OperatorName); } finally { this.reconciliationSemaphore.Release(); } }
/// <summary> /// Adds a feedback loop which initializes a new Appium session to the operator. /// </summary> /// <param name="builder"> /// The operator builder. /// </param> /// <param name="appiumPort"> /// The port at which the Appium server is listening. /// </param> /// <param name="initializer"> /// Optionally, a delegate which can be used to initialize the pod before the session /// is created. /// </param> /// <returns> /// An operator builder which can be used to further configure the operator. /// </returns> public static ChildOperatorBuilder <WebDriverSession, V1Pod> CreatesSession( this ChildOperatorBuilder <WebDriverSession, V1Pod> builder, int appiumPort, SessionPodInitializer initializer = null) { return(builder.PostsFeedback( async(context, cancellationToken) => { var kubernetes = context.Kubernetes; var logger = context.Logger; var session = context.Parent; var pod = context.Child; Feedback <WebDriverSession, V1Pod> feedback = null; if (session?.Spec?.Capabilities == null) { // This is an invalid session; we need at least desired capabilities. logger.LogWarning("Session {session} is missing desired capabilities.", session?.Metadata?.Name); } else if (session.Status?.SessionId != null) { // Do nothing if the session already exists. logger.LogDebug("Session {session} already has a session ID.", session?.Metadata?.Name); } else if (pod?.Status?.Phase != "Running" || !pod.Status.ContainerStatuses.All(c => c.Ready)) { // Do nothing if the pod is not yet ready logger.LogInformation("Not creating a session for session {session} because pod {pod} is not ready yet.", session?.Metadata?.Name, pod?.Metadata?.Name); } else { if (initializer != null) { await initializer(context, cancellationToken).ConfigureAwait(false); } var requestedCapabilities = JsonConvert.DeserializeObject(context.Parent.Spec.Capabilities); var request = JsonConvert.SerializeObject( new { capabilities = requestedCapabilities, }); var content = new StringContent(request, Encoding.UTF8, "application/json"); using (var httpClient = kubernetes.CreatePodHttpClient(context.Child, appiumPort)) using (var remoteResult = await httpClient.PostAsync("wd/hub/session/", content, cancellationToken).ConfigureAwait(false)) { var sessionJson = await remoteResult.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var sessionObject = JObject.Parse(sessionJson); var sessionValue = (JObject)sessionObject.GetValue("value"); feedback = new Feedback <WebDriverSession, V1Pod>() { ParentFeedback = new JsonPatchDocument <WebDriverSession>(), }; if (context.Parent.Status == null) { feedback.ParentFeedback.Add(s => s.Status, new WebDriverSessionStatus()); } // Check whether we should store this as a Kubernetes object. if (sessionValue.TryGetValue("sessionId", out var sessionId)) { feedback.ParentFeedback.Add(s => s.Status.SessionId, sessionId.Value <string>()); feedback.ParentFeedback.Add(s => s.Status.SessionReady, true); feedback.ParentFeedback.Add(s => s.Status.SessionPort, appiumPort); } if (sessionValue.TryGetValue("capabilities", out var capabilities)) { feedback.ParentFeedback.Add(s => s.Status.Capabilities, capabilities.ToString(Formatting.None)); } if (sessionValue.TryGetValue("error", out var error)) { feedback.ParentFeedback.Add(s => s.Status.Error, error.Value <string>()); } if (sessionValue.TryGetValue("message", out var message)) { feedback.ParentFeedback.Add(s => s.Status.Message, message.Value <string>()); } if (sessionValue.TryGetValue("stacktrace", out var stackTrace)) { feedback.ParentFeedback.Add(s => s.Status.StackTrace, stackTrace.Value <string>()); } if (sessionValue.TryGetValue("data", out var data)) { feedback.ParentFeedback.Add(s => s.Status.Data, data.ToString(Formatting.None)); } } } return feedback; })); }
/// <summary> /// Builds an operator which provisions ingress rules for <see cref="WebDriverSession"/> objects which /// use the Fake driver. /// </summary> /// <param name="services"> /// A service provider from which to host services. /// </param> /// <returns> /// A configured operator. /// </returns> public static ChildOperatorBuilder <WebDriverSession, V1Ingress> BuildIngressOperator(IServiceProvider services) { if (services == null) { throw new ArgumentNullException(nameof(services)); } var kubernetes = services.GetRequiredService <KubernetesClient>(); var loggerFactory = services.GetRequiredService <ILoggerFactory>(); var logger = loggerFactory.CreateLogger("FakeOperator"); return(new ChildOperatorBuilder(services) .CreateOperator("WebDriverSession-IngressOperator") .Watches <WebDriverSession>() .Where(s => s.Status?.SessionId != null) .Creates <V1Ingress>( (session, ingress) => { // On the public endpoint, the session id will be the name assigned to the WebDriverSession object. // This is usually a name auto-generated by Kubernetes. // Within the pod running the WebDriver session, the session id will be different - typically a Guid. // Use HTTP request rewriting to patch the session name in the WebDriver URLs. // // URL rewriting is not yet fully standardized within Kubernetes, and this adds a hard dependency on // Traefik as the reverse proxy. // // Traefik was chosen because it's the default reverse proxy in k3s. Adding support for nginx should // not be particularly hard. ingress.EnsureMetadata().EnsureAnnotations(); ingress.Metadata.Annotations.Add(Annotations.RequestModifier, $"ReplacePathRegex: /wd/hub/session/{session.Metadata.Name}/(.*) /wd/hub/session/{session.Status.SessionId}/$1"); ingress.Metadata.Annotations.Add(Annotations.IngressClass, "traefik"); // No need to validate Status.SessionId != null, that's handled by the Where clause above. ingress.Spec = new V1IngressSpec() { Rules = new V1IngressRule[] { new V1IngressRule() { Http = new V1HTTPIngressRuleValue() { Paths = new V1HTTPIngressPath[] { new V1HTTPIngressPath() { Path = $"/wd/hub/session/{session.Metadata.Name}/", PathType = "Prefix", Backend = new V1IngressBackend() { Service = new V1IngressServiceBackend() { Name = session.Metadata.Name, Port = new V1ServiceBackendPort() { Number = session.Status.SessionPort, }, }, }, }, }, }, }, }, }; }) .PostsFeedback((context, cancellationToken) => { Feedback <WebDriverSession, V1Ingress> feedback = null; var session = context.Parent; var ingress = context.Child; if (ingress?.Status?.LoadBalancer?.Ingress == null || ingress.Status.LoadBalancer.Ingress.Count == 0) { logger.LogInformation("Not setting the ingress status to ready for session {session} because the ingress does not have any load balancer endpoints.", context.Parent?.Metadata?.Name); } else if (!session.Status.IngressReady) { feedback = new Feedback <WebDriverSession, V1Ingress>(); feedback.ParentFeedback = new JsonPatchDocument <WebDriverSession>(); feedback.ParentFeedback.Add(s => s.Status.IngressReady, true); } return Task.FromResult(feedback); })); }