/// <summary>
/// Attempts to reconcile a child and parent object.
/// </summary>
/// <param name="parent">
/// The parent object being reconciled.
/// </param>
/// <param name="child">
/// The child object being reconciled, or <see langword="null"/> if a child object does not yet exist.
/// </param>
/// <param name="cancellationToken">
/// A <see cref="CancellationToken"/> which can be used to cancel the asynchronous operation.
/// </param>
/// <returns>
/// A <see cref="Task"/> representing the asynchronous operation.
/// </returns>
public async Task ReconcileAsync(TParent parent, TChild child, CancellationToken cancellationToken)
{
// Let's assume Kubernetes takes care of garbage collection, so the source
// object is always present.
//
// Although children with no parents are possible because of cascading background deletions:
// https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#background-cascading-deletion,
// they should never enter the reconcile loop.
Debug.Assert(parent != null, "Cannot have a child object without a parent object");
if (!await this.reconciliationSemaphore.WaitAsync(0).ConfigureAwait(false))
{
throw new InvalidOperationException("Only one instance of ReconcileAsync can run at a time.");
}
try
{
this.logger.LogInformation(
"Scheduling reconciliation for parent {parent} and child {child} for operator {operatorName}",
parent?.Metadata?.Name,
child?.Metadata?.Name,
this.configuration.OperatorName);
if (child == null)
{
// Create a new object
child = new TChild()
{
Metadata = new V1ObjectMeta()
{
Labels = new Dictionary <string, string>(this.configuration.ChildLabels),
Name = parent.Metadata.Name,
NamespaceProperty = parent.Metadata.NamespaceProperty,
OwnerReferences = new V1OwnerReference[]
{
parent.AsOwnerReference(blockOwnerDeletion: false, controller: false),
},
},
};
child.SetLabel(Annotations.ManagedBy, this.configuration.OperatorName);
this.childFactory(parent, child);
await this.childClient.CreateAsync(child, cancellationToken).ConfigureAwait(false);
}
else
{
this.logger.LogInformation(
"Running {feedbackCount} feedback loops for parent {parent} and child {child} for operator {operatorName} because child is not null.",
this.feedbackLoops.Count,
parent?.Metadata?.Name,
child?.Metadata?.Name,
this.configuration.OperatorName);
Feedback <TParent, TChild> feedback = null;
using (var scope = this.services.CreateScope())
{
var context = new ChildOperatorContext <TParent, TChild>(
parent,
child,
scope.ServiceProvider);
foreach (var feedbackLoop in this.feedbackLoops)
{
if ((feedback = await feedbackLoop(context, cancellationToken).ConfigureAwait(false)) != null)
{
if (feedback.ParentFeedback != null)
{
this.logger.LogInformation(
"Applying patch {feedback} to parent {parent} for operator {operatorName}.",
feedback,
context.Parent?.Metadata?.Name,
this.configuration.OperatorName);
await this.parentClient.PatchAsync(context.Parent, feedback.ParentFeedback, cancellationToken).ConfigureAwait(false);
}
if (feedback.ChildFeedback != null)
{
this.logger.LogInformation(
"Applying patch {feedback} to child {child} for operator {operatorName}.",
feedback,
context.Child?.Metadata?.Name,
this.configuration.OperatorName);
await this.childClient.PatchAsync(context.Child, feedback.ChildFeedback, cancellationToken).ConfigureAwait(false);
}
}
}
}
}
}
catch (Exception ex)
{
this.logger.LogError(ex, "Caught error {errorMessage} while executing reconciliation for operator {operator}", ex.Message, this.configuration.OperatorName);
}
finally
{
this.reconciliationSemaphore.Release();
}
}
/// <summary>
/// Adds a feedback loop which initializes a new Appium session to the operator.
/// </summary>
/// <param name="builder">
/// The operator builder.
/// </param>
/// <param name="appiumPort">
/// The port at which the Appium server is listening.
/// </param>
/// <param name="initializer">
/// Optionally, a delegate which can be used to initialize the pod before the session
/// is created.
/// </param>
/// <returns>
/// An operator builder which can be used to further configure the operator.
/// </returns>
public static ChildOperatorBuilder <WebDriverSession, V1Pod> CreatesSession(
this ChildOperatorBuilder <WebDriverSession, V1Pod> builder,
int appiumPort,
SessionPodInitializer initializer = null)
{
return(builder.PostsFeedback(
async(context, cancellationToken) =>
{
var kubernetes = context.Kubernetes;
var logger = context.Logger;
var session = context.Parent;
var pod = context.Child;
Feedback <WebDriverSession, V1Pod> feedback = null;
if (session?.Spec?.Capabilities == null)
{
// This is an invalid session; we need at least desired capabilities.
logger.LogWarning("Session {session} is missing desired capabilities.", session?.Metadata?.Name);
}
else if (session.Status?.SessionId != null)
{
// Do nothing if the session already exists.
logger.LogDebug("Session {session} already has a session ID.", session?.Metadata?.Name);
}
else if (pod?.Status?.Phase != "Running" || !pod.Status.ContainerStatuses.All(c => c.Ready))
{
// Do nothing if the pod is not yet ready
logger.LogInformation("Not creating a session for session {session} because pod {pod} is not ready yet.", session?.Metadata?.Name, pod?.Metadata?.Name);
}
else
{
if (initializer != null)
{
await initializer(context, cancellationToken).ConfigureAwait(false);
}
var requestedCapabilities = JsonConvert.DeserializeObject(context.Parent.Spec.Capabilities);
var request = JsonConvert.SerializeObject(
new
{
capabilities = requestedCapabilities,
});
var content = new StringContent(request, Encoding.UTF8, "application/json");
using (var httpClient = kubernetes.CreatePodHttpClient(context.Child, appiumPort))
using (var remoteResult = await httpClient.PostAsync("wd/hub/session/", content, cancellationToken).ConfigureAwait(false))
{
var sessionJson = await remoteResult.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var sessionObject = JObject.Parse(sessionJson);
var sessionValue = (JObject)sessionObject.GetValue("value");
feedback = new Feedback <WebDriverSession, V1Pod>()
{
ParentFeedback = new JsonPatchDocument <WebDriverSession>(),
};
if (context.Parent.Status == null)
{
feedback.ParentFeedback.Add(s => s.Status, new WebDriverSessionStatus());
}
// Check whether we should store this as a Kubernetes object.
if (sessionValue.TryGetValue("sessionId", out var sessionId))
{
feedback.ParentFeedback.Add(s => s.Status.SessionId, sessionId.Value <string>());
feedback.ParentFeedback.Add(s => s.Status.SessionReady, true);
feedback.ParentFeedback.Add(s => s.Status.SessionPort, appiumPort);
}
if (sessionValue.TryGetValue("capabilities", out var capabilities))
{
feedback.ParentFeedback.Add(s => s.Status.Capabilities, capabilities.ToString(Formatting.None));
}
if (sessionValue.TryGetValue("error", out var error))
{
feedback.ParentFeedback.Add(s => s.Status.Error, error.Value <string>());
}
if (sessionValue.TryGetValue("message", out var message))
{
feedback.ParentFeedback.Add(s => s.Status.Message, message.Value <string>());
}
if (sessionValue.TryGetValue("stacktrace", out var stackTrace))
{
feedback.ParentFeedback.Add(s => s.Status.StackTrace, stackTrace.Value <string>());
}
if (sessionValue.TryGetValue("data", out var data))
{
feedback.ParentFeedback.Add(s => s.Status.Data, data.ToString(Formatting.None));
}
}
}
return feedback;
}));
}
/// <summary>
/// Builds an operator which provisions ingress rules for <see cref="WebDriverSession"/> objects which
/// use the Fake driver.
/// </summary>
/// <param name="services">
/// A service provider from which to host services.
/// </param>
/// <returns>
/// A configured operator.
/// </returns>
public static ChildOperatorBuilder <WebDriverSession, V1Ingress> BuildIngressOperator(IServiceProvider services)
{
if (services == null)
{
throw new ArgumentNullException(nameof(services));
}
var kubernetes = services.GetRequiredService <KubernetesClient>();
var loggerFactory = services.GetRequiredService <ILoggerFactory>();
var logger = loggerFactory.CreateLogger("FakeOperator");
return(new ChildOperatorBuilder(services)
.CreateOperator("WebDriverSession-IngressOperator")
.Watches <WebDriverSession>()
.Where(s => s.Status?.SessionId != null)
.Creates <V1Ingress>(
(session, ingress) =>
{
// On the public endpoint, the session id will be the name assigned to the WebDriverSession object.
// This is usually a name auto-generated by Kubernetes.
// Within the pod running the WebDriver session, the session id will be different - typically a Guid.
// Use HTTP request rewriting to patch the session name in the WebDriver URLs.
//
// URL rewriting is not yet fully standardized within Kubernetes, and this adds a hard dependency on
// Traefik as the reverse proxy.
//
// Traefik was chosen because it's the default reverse proxy in k3s. Adding support for nginx should
// not be particularly hard.
ingress.EnsureMetadata().EnsureAnnotations();
ingress.Metadata.Annotations.Add(Annotations.RequestModifier, $"ReplacePathRegex: /wd/hub/session/{session.Metadata.Name}/(.*) /wd/hub/session/{session.Status.SessionId}/$1");
ingress.Metadata.Annotations.Add(Annotations.IngressClass, "traefik");
// No need to validate Status.SessionId != null, that's handled by the Where clause above.
ingress.Spec = new V1IngressSpec()
{
Rules = new V1IngressRule[]
{
new V1IngressRule()
{
Http = new V1HTTPIngressRuleValue()
{
Paths = new V1HTTPIngressPath[]
{
new V1HTTPIngressPath()
{
Path = $"/wd/hub/session/{session.Metadata.Name}/",
PathType = "Prefix",
Backend = new V1IngressBackend()
{
Service = new V1IngressServiceBackend()
{
Name = session.Metadata.Name,
Port = new V1ServiceBackendPort()
{
Number = session.Status.SessionPort,
},
},
},
},
},
},
},
},
};
})
.PostsFeedback((context, cancellationToken) =>
{
Feedback <WebDriverSession, V1Ingress> feedback = null;
var session = context.Parent;
var ingress = context.Child;
if (ingress?.Status?.LoadBalancer?.Ingress == null ||
ingress.Status.LoadBalancer.Ingress.Count == 0)
{
logger.LogInformation("Not setting the ingress status to ready for session {session} because the ingress does not have any load balancer endpoints.", context.Parent?.Metadata?.Name);
}
else if (!session.Status.IngressReady)
{
feedback = new Feedback <WebDriverSession, V1Ingress>();
feedback.ParentFeedback = new JsonPatchDocument <WebDriverSession>();
feedback.ParentFeedback.Add(s => s.Status.IngressReady, true);
}
return Task.FromResult(feedback);
}));
}
