public Cluster(ClusterStatus status, Cluster copyFrom)
: this(copyFrom.InternalName,
status,
copyFrom.AppCount,
copyFrom.ServiceCount,
copyFrom.Address,
new List<int>(copyFrom.Ports),
new List<ClusterUser>(copyFrom.Users),
copyFrom.CreatedOn)
{
}
private async Task<Cluster> ProcessDeletingClusterAsync(Cluster cluster)
{
ClusterOperationStatus deleteStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (deleteStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
return new Cluster(ClusterStatus.Remove, cluster); // hopefully shouldn't ever get here
case ClusterOperationStatus.Deleting:
return cluster;
case ClusterOperationStatus.ClusterNotFound:
ServiceEventSource.Current.ServiceMessage(this, "Cluster successfully deleted: {0}.", cluster.Address);
return new Cluster(ClusterStatus.Deleted, cluster);
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to delete: {0}.", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
}
return cluster;
}
private async Task<Cluster> ProcessReadyClusterAsync(Cluster cluster)
{
if (DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime() >= this.config.MaximumClusterUptime)
{
ServiceEventSource.Current.ServiceMessage(this, "Cluster expired: {0}", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
}
ClusterOperationStatus readyStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (readyStatus)
{
case ClusterOperationStatus.Deleting:
return new Cluster(ClusterStatus.Deleting, cluster);
}
try
{
int deployedApplications = await this.applicationDeployService.GetApplicationCountAsync(cluster.Address, ClusterConnectionPort);
int deployedServices = await this.applicationDeployService.GetServiceCountAsync(cluster.Address, ClusterConnectionPort);
return new Cluster(
cluster.InternalName,
cluster.Status,
deployedApplications,
deployedServices,
cluster.Address,
cluster.Ports,
cluster.Users,
cluster.CreatedOn);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Unable to determine application and service count. Cluster: {0}. Error: {1}",
cluster.Address,
e.GetActualMessage());
}
return cluster;
}
private async Task<Cluster> ProcessRemoveClusterAsync(Cluster cluster)
{
ClusterOperationStatus removeStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (removeStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
ServiceEventSource.Current.ServiceMessage(this, "Deleting cluster {0}.", cluster.Address);
await this.clusterOperator.DeleteClusterAsync(cluster.InternalName);
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.Deleting:
return new Cluster(ClusterStatus.Deleting, cluster);
case ClusterOperationStatus.ClusterNotFound:
return new Cluster(ClusterStatus.Deleted, cluster);
}
return cluster;
}
private async Task<Cluster> ProcessCreatingClusterAsync(Cluster cluster)
{
ClusterOperationStatus creatingStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (creatingStatus)
{
case ClusterOperationStatus.Creating:
return cluster;
case ClusterOperationStatus.Ready:
IEnumerable<int> ports = await this.clusterOperator.GetClusterPortsAsync(cluster.InternalName);
try
{
await this.applicationDeployService.QueueApplicationDeploymentAsync(cluster.Address, ClusterConnectionPort);
}
catch (Exception e)
{
// couldn't queue samples for deployment, but that shouldn't prevent starting the cluster.
ServiceEventSource.Current.ServiceMessage(
this,
"Failed to queue sample deployment. Cluster: {0} Error: {1}",
cluster.Address,
e.GetActualMessage());
}
ServiceEventSource.Current.ServiceMessage(
this,
"Cluster is ready: {0} with ports: {1}",
cluster.Address,
String.Join(",", cluster.Ports));
return new Cluster(
cluster.InternalName,
ClusterStatus.Ready,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
new List<int>(ports),
new List<ClusterUser>(cluster.Users),
DateTimeOffset.UtcNow);
case ClusterOperationStatus.CreateFailed:
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to create: {0}", cluster.Address);
return new Cluster(ClusterStatus.Remove, cluster);
case ClusterOperationStatus.Deleting:
return new Cluster(ClusterStatus.Deleting, cluster);
default:
return cluster;
}
}
private async Task<Cluster> ProcessNewClusterAsync(Cluster cluster)
{
try
{
string address = await this.clusterOperator.CreateClusterAsync(cluster.InternalName);
ServiceEventSource.Current.ServiceMessage(this, "Creating cluster: {0}", address);
return new Cluster(
cluster.InternalName,
ClusterStatus.Creating,
cluster.AppCount,
cluster.ServiceCount,
address,
new List<int>(cluster.Ports),
new List<ClusterUser>(cluster.Users),
cluster.CreatedOn);
}
catch (InvalidOperationException e)
{
// cluster with this name might already exist, so remove this one.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to create: {0}. {1}", cluster.Address, e.Message);
// mark as deleted so it gets removed from the list.
return new Cluster(ClusterStatus.Deleted, cluster);
}
}
/// <summary>
/// Processes a cluster based on its current state.
/// </summary>
/// <returns></returns>
internal Task<Cluster> ProcessClusterStatusAsync(Cluster cluster)
{
switch (cluster.Status)
{
case ClusterStatus.New:
return this.ProcessNewClusterAsync(cluster);
case ClusterStatus.Creating:
return this.ProcessCreatingClusterAsync(cluster);
case ClusterStatus.Ready:
return this.ProcessReadyClusterAsync(cluster);
case ClusterStatus.Remove:
return this.ProcessRemoveClusterAsync(cluster);
case ClusterStatus.Deleting:
return this.ProcessDeletingClusterAsync(cluster);
default:
return Task.FromResult(cluster);
}
}
/// <summary>
/// Processes a request to join a cluster.
/// </summary>
/// <param name="clusterId"></param>
/// <param name="user"></param>
/// <returns></returns>
public async Task JoinClusterAsync(int clusterId, string userEmail)
{
if (String.IsNullOrWhiteSpace(userEmail))
{
throw new ArgumentNullException("userEmail");
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request. Cluster: {0}.", clusterId);
IReliableDictionary<int, Cluster> clusterDictionary =
await this.StateManager.GetOrAddAsync<IReliableDictionary<int, Cluster>>(ClusterDictionaryName);
foreach (KeyValuePair<int, Cluster> item in clusterDictionary)
{
if (item.Value.Users.Any(x => String.Equals(x.Email, userEmail, StringComparison.OrdinalIgnoreCase)))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. User already exists on cluster: {0}.",
item.Key);
throw new JoinClusterFailedException(JoinClusterFailedReason.UserAlreadyJoined);
}
}
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalResult<Cluster> result = await clusterDictionary.TryGetValueAsync(tx, clusterId, LockMode.Update);
if (!result.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster does not exist. Cluster ID: {0}.",
clusterId);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterDoesNotExist);
}
Cluster cluster = result.Value;
// make sure the cluster isn't about to be deleted.
if ((DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime()) > (this.config.MaximumClusterUptime))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster has expired. Cluster: {0}. Cluster creation time: {1}",
clusterId,
cluster.CreatedOn.ToUniversalTime());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterExpired);
}
// make sure the cluster is ready
if (cluster.Status != ClusterStatus.Ready)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is not ready. Cluster: {0}. Status: {1}",
clusterId,
cluster.Status);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterNotReady);
}
if (cluster.Users.Count() >= this.config.MaximumUsersPerCluster)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is full. Cluster: {0}. Users: {1}",
clusterId,
cluster.Users.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterFull);
}
int userPort;
string clusterAddress = cluster.Address;
TimeSpan clusterTimeRemaining = this.config.MaximumClusterUptime - (DateTimeOffset.UtcNow - cluster.CreatedOn);
DateTimeOffset clusterExpiration = cluster.CreatedOn + this.config.MaximumClusterUptime;
try
{
userPort = cluster.Ports.First(port => !cluster.Users.Any(x => x.Port == port));
}
catch (InvalidOperationException)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. No available ports. Cluster: {0}. Users: {1}. Ports: {2}",
clusterId,
cluster.Users.Count(),
cluster.Ports.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.NoPortsAvailable);
}
try
{
ServiceEventSource.Current.ServiceMessage(this, "Sending join mail. Cluster: {0}.", clusterId);
List<HyperlinkView> links = new List<HyperlinkView>();
links.Add(new HyperlinkView("http://" + clusterAddress + ":" + ClusterHttpGatewayPort + "/Explorer/index.html", "Service Fabric Explorer", "explore what's on the cluster with the built-in Service Fabric Explorer."));
try
{
IEnumerable<ApplicationView> applications = await this.applicationDeployService.GetApplicationDeploymentsAsync(cluster.Address, ClusterConnectionPort);
links.AddRange(applications.Select(x => x.EntryServiceInfo));
}
catch(Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to get application deployment info. {0}.", e.GetActualMessage());
}
await this.mailer.SendJoinMail(
userEmail,
clusterAddress + ":" + ClusterConnectionPort,
userPort,
clusterTimeRemaining,
clusterExpiration,
links);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to send join mail. {0}.", e.GetActualMessage());
throw new JoinClusterFailedException(JoinClusterFailedReason.SendMailFailed);
}
List<ClusterUser> newUserList = new List<ClusterUser>(cluster.Users);
newUserList.Add(new ClusterUser(userEmail, userPort));
Cluster updatedCluster = new Cluster(
cluster.InternalName,
cluster.Status,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
cluster.Ports,
newUserList,
cluster.CreatedOn);
await clusterDictionary.SetAsync(tx, clusterId, updatedCluster);
await tx.CommitAsync();
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request completed. Cluster: {0}.", clusterId);
}
/// <summary>
/// Processes a cluster based on its current state.
/// </summary>
/// <returns></returns>
internal async Task ProcessClusterStatusAsync(Cluster cluster)
{
switch (cluster.Status)
{
case ClusterStatus.New:
Random random = new Random();
cluster.Address = await this.clusterOperator.CreateClusterAsync(random.Next().ToString());
cluster.Status = ClusterStatus.Creating;
break;
case ClusterStatus.Creating:
ClusterOperationStatus creatingStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (creatingStatus)
{
case ClusterOperationStatus.Creating:
// still creating
break;
case ClusterOperationStatus.Ready:
cluster.Ports = await this.clusterOperator.GetClusterPortsAsync(cluster.Address);
cluster.CreatedOn = DateTimeOffset.UtcNow;
cluster.Status = ClusterStatus.Ready;
break;
case ClusterOperationStatus.CreateFailed:
cluster.Status = ClusterStatus.New;
break;
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
break;
case ClusterStatus.Ready:
if (DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime() >= this.Config.MaxClusterUptime)
{
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
cluster.Status = ClusterStatus.Deleting;
}
ClusterOperationStatus readyStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (readyStatus)
{
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
//TODO: update application and service count
break;
case ClusterStatus.Remove:
ClusterOperationStatus removeStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (removeStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
cluster.Status = ClusterStatus.Deleting;
break;
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
break;
case ClusterStatus.Deleting:
ClusterOperationStatus deleteStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (deleteStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
break;
case ClusterOperationStatus.Deleting:
break; // still in progress
case ClusterOperationStatus.ClusterNotFound:
cluster.Status = ClusterStatus.Deleted;
break;
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
cluster.Status = ClusterStatus.Remove;
break;
}
break;
}
}