internal async Task <bool> TryDequeueAndProcessAsync(CancellationToken cancellationToken)
{
IReliableQueue <Guid> queue =
await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName);
IReliableDictionary <Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <Guid> workItem = await queue.TryDequeueAsync(tx, this.transactionTimeout, cancellationToken);
if (!workItem.HasValue)
{
ServiceEventSource.Current.ServiceMessage(this, "No new application deployment requests.");
return(false);
}
Guid workItemId = workItem.Value;
ConditionalValue <ApplicationDeployment> appDeployment =
await dictionary.TryGetValueAsync(tx, workItemId, LockMode.Update, this.transactionTimeout, cancellationToken);
if (!appDeployment.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Found queued application deployment request with no associated deployment information. Discarding.");
return(true);
}
ApplicationDeployment processedDeployment = await this.ProcessApplicationDeployment(appDeployment.Value, cancellationToken);
if (processedDeployment.Status == ApplicationDeployStatus.Complete ||
processedDeployment.Status == ApplicationDeployStatus.Failed)
{
// Remove deployments that completed or failed
await dictionary.TryRemoveAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
}
else
{
// The deployment hasn't completed or failed, so queue up the next stage of deployment
await queue.EnqueueAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
// And update the deployment record with the new status
await dictionary.SetAsync(tx, workItemId, processedDeployment, this.transactionTimeout, cancellationToken);
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment request successfully processed. Cluster: {0}. Status: {1}",
processedDeployment.Cluster,
processedDeployment.Status);
}
await tx.CommitAsync();
}
return(true);
}
public ApplicationDeployment(ApplicationDeployStatus status, ApplicationDeployment copyFrom)
: this(
copyFrom.Cluster,
status,
copyFrom.ImageStorePath,
copyFrom.ApplicationTypeName,
copyFrom.ApplicationTypeVersion,
copyFrom.ApplicationInstanceName,
copyFrom.PackageZipFilePath,
copyFrom.DeploymentTimestamp)
{
}
public ApplicationDeployment(ApplicationDeployStatus status, ApplicationDeployment copyFrom)
: this(
copyFrom.Cluster,
status,
copyFrom.ImageStorePath,
copyFrom.ApplicationTypeName,
copyFrom.ApplicationTypeVersion,
copyFrom.ApplicationInstanceName,
copyFrom.PackageZipFilePath,
copyFrom.DeploymentTimestamp)
{
}
/// <summary>
/// Queues deployment of application packages to the given cluster.
/// </summary>
/// <param name="clusterAddress"></param>
/// <param name="clusterPort"></param>
/// <returns></returns>
public async Task <IEnumerable <Guid> > QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort)
{
IReliableQueue <Guid> queue =
await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName);
IReliableDictionary <Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName);
List <Guid> workIds = new List <Guid>(this.ApplicationPackages.Count());
using (ITransaction tx = this.StateManager.CreateTransaction())
{
// Grab each application package that's included with the service
// and create an ApplicationDeployment record of it.
// Then queue a job to begin processing each one.
foreach (ApplicationPackageInfo package in this.ApplicationPackages)
{
Guid id = Guid.NewGuid();
ApplicationDeployment applicationDeployment = new ApplicationDeployment(
cluster: GetClusterAddress(clusterAddress, clusterPort),
status: ApplicationDeployStatus.Copy,
imageStorePath: null,
applicationTypeName: package.ApplicationTypeName,
applicationTypeVersion: package.ApplicationTypeVersion,
applicationInstanceName: GetApplicationInstanceName(package.PackageFileName),
packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName),
timestamp: DateTimeOffset.UtcNow);
await dictionary.AddAsync(tx, id, applicationDeployment);
await queue.EnqueueAsync(tx, id);
workIds.Add(id);
}
await tx.CommitAsync();
return(workIds);
}
}
internal async Task <ApplicationDeployment> ProcessApplicationDeployment(
ApplicationDeployment applicationDeployment, CancellationToken cancellationToken)
{
try
{
switch (applicationDeployment.Status)
{
case ApplicationDeployStatus.Copy:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Copying to image store. Cluster: {0}. Application: {1}. Package path: {2}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath);
try
{
// Unzip the package contents to a temporary location on disk.
if (!this.applicationPackageTempDirectory.Exists)
{
this.applicationPackageTempDirectory.Create();
}
FileInfo packageFile = new FileInfo(applicationDeployment.PackageZipFilePath);
DirectoryInfo packageExtractLocation =
new DirectoryInfo(Path.Combine(this.applicationPackageTempDirectory.FullName, applicationDeployment.ApplicationInstanceName));
if (packageExtractLocation.Exists)
{
packageExtractLocation.Delete(true);
}
ServiceEventSource.Current.ServiceMessage(
this,
"Extracting application package from {0} to {1}",
packageFile.FullName,
packageExtractLocation.FullName);
ZipFile.ExtractToDirectory(packageFile.FullName, packageExtractLocation.FullName);
// Copy the unzipped application package to the cluster's imagestore and clean up.
string imageStorePath = await this.applicationOperator.CopyPackageToImageStoreAsync(
applicationDeployment.Cluster,
packageExtractLocation.FullName,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
cancellationToken);
packageExtractLocation.Delete(true);
return(new ApplicationDeployment(
applicationDeployment.Cluster,
ApplicationDeployStatus.Register,
imageStorePath,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
applicationDeployment.DeploymentTimestamp));
}
catch (FabricServiceNotFoundException fsnfe)
{
// image store service isn't ready yet.
// This is retry-able, just need to wait a bit for it to come up.
// This can happen when an application deployment is attempted immediately after a cluster comes up.
ServiceEventSource.Current.ApplicationDeploymentFailureCopyFailure(
applicationDeployment.Cluster,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
fsnfe.Message);
return(applicationDeployment);
}
catch (FileNotFoundException fnfe)
{
ServiceEventSource.Current.ApplicationDeploymentFailureCorruptPackage(
applicationDeployment.Cluster,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
fnfe.Message);
return(new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment));
}
case ApplicationDeployStatus.Register:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Registering. Cluster: {0}. Application: {1}, Imagestore path: {2}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.ImageStorePath);
try
{
await this.applicationOperator.RegisterApplicationAsync(
applicationDeployment.Cluster,
applicationDeployment.ImageStorePath,
cancellationToken);
return(new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment));
}
catch (FabricElementAlreadyExistsException)
{
ServiceEventSource.Current.ApplicationDeploymentFailureAlreadyRegistered(
applicationDeployment.Cluster,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath);
// application already exists, set status to Create it so we don't keep failing on this.
return(new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment));
}
case ApplicationDeployStatus.Create:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Creating. Cluster: {0}. Application: {1}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName);
await this.applicationOperator.CreateApplicationAsync(
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
cancellationToken);
return(new ApplicationDeployment(ApplicationDeployStatus.Complete, applicationDeployment));
default:
return(applicationDeployment);
}
}
catch (FabricTransientException fte)
{
ServiceEventSource.Current.ApplicationDeploymentFailureTransientError(
applicationDeployment.Cluster,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
Enum.GetName(typeof(ApplicationDeployStatus), applicationDeployment.Status),
fte.Message);
// return the deployment unchanged. It will be returned to the queue and tried again.
return(applicationDeployment);
}
catch (Exception e)
{
ServiceEventSource.Current.ApplicationDeploymentFailureUnknownError(
applicationDeployment.Cluster,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
e.ToString());
return(new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment));
}
}
/// <summary>
/// Queues deployment of application packages to the given cluster.
/// </summary>
/// <param name="clusterAddress"></param>
/// <param name="clusterPort"></param>
/// <returns></returns>
public async Task<IEnumerable<Guid>> QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort)
{
IReliableQueue<Guid> queue =
await this.StateManager.GetOrAddAsync<IReliableQueue<Guid>>(QueueName);
IReliableDictionary<Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync<IReliableDictionary<Guid, ApplicationDeployment>>(DictionaryName);
List<Guid> workIds = new List<Guid>(this.ApplicationPackages.Count());
using (ITransaction tx = this.StateManager.CreateTransaction())
{
// Grab each application package that's included with the service
// and create an ApplicationDeployment record of it.
// Then queue a job to begin processing each one.
foreach (ApplicationPackageInfo package in this.ApplicationPackages)
{
Guid id = Guid.NewGuid();
ApplicationDeployment applicationDeployment = new ApplicationDeployment(
cluster: GetClusterAddress(clusterAddress, clusterPort),
status: ApplicationDeployStatus.Copy,
imageStorePath: null,
applicationTypeName: package.ApplicationTypeName,
applicationTypeVersion: package.ApplicationTypeVersion,
applicationInstanceName: GetApplicationInstanceName(package.PackageFileName),
packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName),
timestamp: DateTimeOffset.UtcNow);
await dictionary.AddAsync(tx, id, applicationDeployment);
await queue.EnqueueAsync(tx, id);
workIds.Add(id);
}
await tx.CommitAsync();
return workIds;
}
}
internal async Task<ApplicationDeployment> ProcessApplicationDeployment(
ApplicationDeployment applicationDeployment, CancellationToken cancellationToken)
{
try
{
switch (applicationDeployment.Status)
{
case ApplicationDeployStatus.Copy:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Copying to image store. Cluster: {0}. Application: {1}. Package path: {2}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath);
try
{
// Unzip the package contents to a temporary location on disk.
if (!this.applicationPackageTempDirectory.Exists)
{
this.applicationPackageTempDirectory.Create();
}
FileInfo packageFile = new FileInfo(applicationDeployment.PackageZipFilePath);
DirectoryInfo packageExtractLocation =
new DirectoryInfo(Path.Combine(this.applicationPackageTempDirectory.FullName, applicationDeployment.ApplicationInstanceName));
if (packageExtractLocation.Exists)
{
packageExtractLocation.Delete(true);
}
ServiceEventSource.Current.ServiceMessage(
this,
"Extracting application package from {0} to {1}",
packageFile.FullName,
packageExtractLocation.FullName);
ZipFile.ExtractToDirectory(packageFile.FullName, packageExtractLocation.FullName);
// Copy the unzipped application package to the cluster's imagestore and clean up.
string imageStorePath = await this.applicationOperator.CopyPackageToImageStoreAsync(
applicationDeployment.Cluster,
packageExtractLocation.FullName,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
cancellationToken);
packageExtractLocation.Delete(true);
return new ApplicationDeployment(
applicationDeployment.Cluster,
ApplicationDeployStatus.Register,
imageStorePath,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.PackageZipFilePath,
applicationDeployment.DeploymentTimestamp);
}
catch (FabricServiceNotFoundException fsnfe)
{
// image store service isn't ready yet.
// This is retry-able, just need to wait a bit for it to come up.
// This can happen when an application deployment is attempted immediately after a cluster comes up.
ServiceEventSource.Current.ServiceMessage(
this,
"Copy to image store failed with FabricServiceNotFoundException. Package: {0}. Error: {1}",
applicationDeployment.PackageZipFilePath,
fsnfe.Message);
return applicationDeployment;
}
catch (FileNotFoundException fnfe)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Found corrupt application package. Package: {0}. Error: {1}",
applicationDeployment.PackageZipFilePath,
fnfe.Message);
return new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment);
}
case ApplicationDeployStatus.Register:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Registering. Cluster: {0}. Application: {1}, Imagestore path: {2}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.ImageStorePath);
try
{
await this.applicationOperator.RegisterApplicationAsync(
applicationDeployment.Cluster,
applicationDeployment.ImageStorePath,
cancellationToken);
return new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment);
}
catch (FabricElementAlreadyExistsException)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Application package is already registered. Package: {0}.",
applicationDeployment.PackageZipFilePath);
// application already exists, set status to Create it so we don't keep failing on this.
return new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment);
}
case ApplicationDeployStatus.Create:
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment: Creating. Cluster: {0}. Application: {1}",
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName);
await this.applicationOperator.CreateApplicationAsync(
applicationDeployment.Cluster,
applicationDeployment.ApplicationInstanceName,
applicationDeployment.ApplicationTypeName,
applicationDeployment.ApplicationTypeVersion,
cancellationToken);
return new ApplicationDeployment(ApplicationDeployStatus.Complete, applicationDeployment);
default:
return applicationDeployment;
}
}
catch (FabricTransientException fte)
{
ServiceEventSource.Current.ServiceMessage(
this,
"A transient error occured during package processing. Package: {0}. Stage: {1}. Error: {2}",
applicationDeployment.ApplicationInstanceName,
applicationDeployment.Status,
fte.Message);
// return the deployment unchanged. It will be returned to the queue and tried again.
return applicationDeployment;
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Application package processing failed. Package: {0}. Error: {1}",
applicationDeployment.PackageZipFilePath,
e.ToString());
return new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment);
}
}
