/// <summary>
/// Create a new instance copied from the given Cluster with the given status.
/// </summary>
/// <param name="status"></param>
/// <param name="copyFrom"></param>
public Cluster(ClusterStatus status, Cluster copyFrom)
: this(copyFrom.InternalName,
status,
copyFrom.AppCount,
copyFrom.ServiceCount,
copyFrom.Address,
new List<int>(copyFrom.Ports),
new List<ClusterUser>(copyFrom.Users),
copyFrom.CreatedOn)
{
}
/// <summary>
/// Processes a cluster that is in the "Deleting" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task<Cluster> ProcessDeletingClusterAsync(Cluster cluster)
{
ClusterOperationStatus deleteStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (deleteStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
// hopefully shouldn't ever get here
return new Cluster(ClusterStatus.Remove, cluster);
case ClusterOperationStatus.Deleting:
// still deleting, no updates necessary
return cluster;
case ClusterOperationStatus.ClusterNotFound:
// If the cluster can't be found, it's been deleted.
ServiceEventSource.Current.ServiceMessage(this, "Cluster successfully deleted: {0}.", cluster.Address);
return new Cluster(ClusterStatus.Deleted, cluster);
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
// Failed to delete, set its status to "remove" to try again.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to delete: {0}.", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
}
return cluster;
}
/// <summary>
/// Processes a cluster in the "Remove" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task<Cluster> ProcessRemoveClusterAsync(Cluster cluster)
{
ClusterOperationStatus removeStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (removeStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
// In any of these cases, instruct the operator to delete the cluster.
ServiceEventSource.Current.ServiceMessage(this, "Deleting cluster {0}.", cluster.Address);
await this.clusterOperator.DeleteClusterAsync(cluster.InternalName);
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.Deleting:
// If the cluster is now being deleted, update the status accordingly.
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.ClusterNotFound:
// Cluster was already deleted.
return new Cluster(ClusterStatus.Deleted, cluster);
}
return cluster;
}
/// <summary>
/// Processes clusters in the "Ready" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task<Cluster> ProcessReadyClusterAsync(Cluster cluster)
{
// Check for expiration. If the cluster has expired, mark it for removal.
if (DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime() >= this.config.MaximumClusterUptime)
{
ServiceEventSource.Current.ServiceMessage(this, "Cluster expired: {0}", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
}
ClusterOperationStatus readyStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (readyStatus)
{
case ClusterOperationStatus.Deleting:
// If the cluster was deleted, mark the state accordingly
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.ClusterNotFound:
// Cluster was already deleted.
return new Cluster(ClusterStatus.Deleted, cluster);
}
try
{
// Update the application and service count for the cluster.
int deployedApplications = await this.applicationDeployService.GetApplicationCountAsync(cluster.Address, ClusterConnectionPort);
int deployedServices = await this.applicationDeployService.GetServiceCountAsync(cluster.Address, ClusterConnectionPort);
return new Cluster(
cluster.InternalName,
cluster.Status,
deployedApplications,
deployedServices,
cluster.Address,
cluster.Ports,
cluster.Users,
cluster.CreatedOn);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Unable to determine application and service count. Cluster: {0}. Error: {1}",
cluster.Address,
e.GetActualMessage());
}
return cluster;
}
/// <summary>
/// Processes a cluster in the "Creating" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task<Cluster> ProcessCreatingClusterAsync(Cluster cluster)
{
ClusterOperationStatus creatingStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (creatingStatus)
{
case ClusterOperationStatus.Creating:
// Still creating, no updates necessary.
return cluster;
case ClusterOperationStatus.Ready:
// Cluster is ready to go.
try
{
// Queue up sample application deployment
await this.applicationDeployService.QueueApplicationDeploymentAsync(cluster.Address, ClusterConnectionPort);
}
catch (Exception e)
{
// couldn't queue samples for deployment, but that shouldn't prevent starting the cluster.
ServiceEventSource.Current.ServiceMessage(
this,
"Failed to queue sample deployment. Cluster: {0} Error: {1}",
cluster.Address,
e.GetActualMessage());
}
ServiceEventSource.Current.ServiceMessage(
this,
"Cluster is ready: {0}",
cluster.Address);
return new Cluster(
cluster.InternalName,
ClusterStatus.Ready,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
cluster.Ports,
new List<ClusterUser>(cluster.Users),
DateTimeOffset.UtcNow);
case ClusterOperationStatus.CreateFailed:
// Failed to create the cluster, so remove it.
// Processing will add a new one in the next iteration if we need more.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to create: {0}", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
case ClusterOperationStatus.Deleting:
// Cluster is being deleted.
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.ClusterNotFound:
// Cluster was deleted before it finished being created.
return new Cluster(ClusterStatus.Deleted, cluster);
default:
return cluster;
}
}
/// <summary>
/// Processes a new cluster.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task<Cluster> ProcessNewClusterAsync(Cluster cluster)
{
try
{
List<int> ports = new List<int>();
Random random = new Random();
for (int i = 0; i < 5; ++i)
{
ports.Add(random.Next(8000, 9000));
}
string address = await this.clusterOperator.CreateClusterAsync(cluster.InternalName, ports);
ServiceEventSource.Current.ServiceMessage(this, "Creating cluster: {0} with ports {1}",
address,
String.Join(",", ports));
return new Cluster(
cluster.InternalName,
ClusterStatus.Creating,
cluster.AppCount,
cluster.ServiceCount,
address,
ports,
new List<ClusterUser>(cluster.Users),
cluster.CreatedOn);
}
catch (InvalidOperationException e)
{
// cluster with this name might already exist, so remove this one.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to create: {0}. {1}", cluster.Address, e.Message);
// mark as deleted so it gets removed from the list.
return new Cluster(ClusterStatus.Deleted, cluster);
}
}
/// <summary>
/// Processes a cluster based on its current state.
/// </summary>
/// <returns></returns>
internal Task<Cluster> ProcessClusterStatusAsync(Cluster cluster)
{
switch (cluster.Status)
{
case ClusterStatus.New:
return this.ProcessNewClusterAsync(cluster);
case ClusterStatus.Creating:
return this.ProcessCreatingClusterAsync(cluster);
case ClusterStatus.Ready:
return this.ProcessReadyClusterAsync(cluster);
case ClusterStatus.Remove:
return this.ProcessRemoveClusterAsync(cluster);
case ClusterStatus.Deleting:
return this.ProcessDeletingClusterAsync(cluster);
default:
return Task.FromResult(cluster);
}
}
/// <summary>
/// Processes a request to join a cluster.
/// </summary>
/// <param name="clusterId"></param>
/// <param name="user"></param>
/// <returns></returns>
public async Task JoinClusterAsync(int clusterId, string userEmail)
{
if (String.IsNullOrWhiteSpace(userEmail))
{
throw new ArgumentNullException("userEmail");
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request. Cluster: {0}.", clusterId);
IReliableDictionary<int, Cluster> clusterDictionary =
await this.StateManager.GetOrAddAsync<IReliableDictionary<int, Cluster>>(ClusterDictionaryName);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable<KeyValuePair<int, Cluster>> clusterAsyncEnumerable = await clusterDictionary.CreateEnumerableAsync(tx);
await clusterAsyncEnumerable.ForeachAsync(
CancellationToken.None,
item =>
{
if (item.Value.Users.Any(x => String.Equals(x.Email, userEmail, StringComparison.OrdinalIgnoreCase)))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. User already exists on cluster: {0}.",
item.Key);
throw new JoinClusterFailedException(JoinClusterFailedReason.UserAlreadyJoined);
}
});
ConditionalValue<Cluster> result = await clusterDictionary.TryGetValueAsync(tx, clusterId, LockMode.Update);
if (!result.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster does not exist. Cluster ID: {0}.",
clusterId);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterDoesNotExist);
}
Cluster cluster = result.Value;
// make sure the cluster isn't about to be deleted.
if ((DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime()) > (this.config.MaximumClusterUptime))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster has expired. Cluster: {0}. Cluster creation time: {1}",
clusterId,
cluster.CreatedOn.ToUniversalTime());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterExpired);
}
// make sure the cluster is ready
if (cluster.Status != ClusterStatus.Ready)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is not ready. Cluster: {0}. Status: {1}",
clusterId,
cluster.Status);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterNotReady);
}
if (cluster.Users.Count() >= this.config.MaximumUsersPerCluster)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is full. Cluster: {0}. Users: {1}",
clusterId,
cluster.Users.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterFull);
}
int userPort;
string clusterAddress = cluster.Address;
TimeSpan clusterTimeRemaining = this.config.MaximumClusterUptime - (DateTimeOffset.UtcNow - cluster.CreatedOn);
DateTimeOffset clusterExpiration = cluster.CreatedOn + this.config.MaximumClusterUptime;
try
{
userPort = cluster.Ports.First(port => !cluster.Users.Any(x => x.Port == port));
}
catch (InvalidOperationException)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. No available ports. Cluster: {0}. Users: {1}. Ports: {2}",
clusterId,
cluster.Users.Count(),
cluster.Ports.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.NoPortsAvailable);
}
try
{
ServiceEventSource.Current.ServiceMessage(this, "Sending join mail. Cluster: {0}.", clusterId);
List<HyperlinkView> links = new List<HyperlinkView>();
links.Add(
new HyperlinkView(
"http://" + clusterAddress + ":" + ClusterHttpGatewayPort + "/Explorer/index.html",
"Service Fabric Explorer",
"explore what's on the cluster with the built-in Service Fabric Explorer."));
try
{
IEnumerable<ApplicationView> applications =
await this.applicationDeployService.GetApplicationDeploymentsAsync(cluster.Address, ClusterConnectionPort);
links.AddRange(applications.Select(x => x.EntryServiceInfo));
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to get application deployment info. {0}.", e.GetActualMessage());
}
await this.mailer.SendJoinMail(
userEmail,
clusterAddress + ":" + ClusterConnectionPort,
userPort,
clusterTimeRemaining,
clusterExpiration,
links);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to send join mail. {0}.", e.GetActualMessage());
throw new JoinClusterFailedException(JoinClusterFailedReason.SendMailFailed);
}
List<ClusterUser> newUserList = new List<ClusterUser>(cluster.Users);
newUserList.Add(new ClusterUser(userEmail, userPort));
Cluster updatedCluster = new Cluster(
cluster.InternalName,
cluster.Status,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
cluster.Ports,
newUserList,
cluster.CreatedOn);
await clusterDictionary.SetAsync(tx, clusterId, updatedCluster);
await tx.CommitAsync();
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request completed. Cluster: {0}.", clusterId);
}
