internal async Task <bool> TryDequeueAndProcessAsync(CancellationToken cancellationToken) { IReliableQueue <Guid> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName); IReliableDictionary <Guid, ApplicationDeployment> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName); using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <Guid> workItem = await queue.TryDequeueAsync(tx, this.transactionTimeout, cancellationToken); if (!workItem.HasValue) { ServiceEventSource.Current.ServiceMessage(this, "No new application deployment requests."); return(false); } Guid workItemId = workItem.Value; ConditionalValue <ApplicationDeployment> appDeployment = await dictionary.TryGetValueAsync(tx, workItemId, LockMode.Update, this.transactionTimeout, cancellationToken); if (!appDeployment.HasValue) { ServiceEventSource.Current.ServiceMessage( this, "Found queued application deployment request with no associated deployment information. Discarding."); return(true); } ApplicationDeployment processedDeployment = await this.ProcessApplicationDeployment(appDeployment.Value, cancellationToken); if (processedDeployment.Status == ApplicationDeployStatus.Complete || processedDeployment.Status == ApplicationDeployStatus.Failed) { // Remove deployments that completed or failed await dictionary.TryRemoveAsync(tx, workItemId, this.transactionTimeout, cancellationToken); } else { // The deployment hasn't completed or failed, so queue up the next stage of deployment await queue.EnqueueAsync(tx, workItemId, this.transactionTimeout, cancellationToken); // And update the deployment record with the new status await dictionary.SetAsync(tx, workItemId, processedDeployment, this.transactionTimeout, cancellationToken); ServiceEventSource.Current.ServiceMessage( this, "Application deployment request successfully processed. Cluster: {0}. Status: {1}", processedDeployment.Cluster, processedDeployment.Status); } await tx.CommitAsync(); } return(true); }
public ApplicationDeployment(ApplicationDeployStatus status, ApplicationDeployment copyFrom) : this( copyFrom.Cluster, status, copyFrom.ImageStorePath, copyFrom.ApplicationTypeName, copyFrom.ApplicationTypeVersion, copyFrom.ApplicationInstanceName, copyFrom.PackageZipFilePath, copyFrom.DeploymentTimestamp) { }
/// <summary> /// Queues deployment of application packages to the given cluster. /// </summary> /// <param name="clusterAddress"></param> /// <param name="clusterPort"></param> /// <returns></returns> public async Task <IEnumerable <Guid> > QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort) { IReliableQueue <Guid> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName); IReliableDictionary <Guid, ApplicationDeployment> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName); List <Guid> workIds = new List <Guid>(this.ApplicationPackages.Count()); using (ITransaction tx = this.StateManager.CreateTransaction()) { // Grab each application package that's included with the service // and create an ApplicationDeployment record of it. // Then queue a job to begin processing each one. foreach (ApplicationPackageInfo package in this.ApplicationPackages) { Guid id = Guid.NewGuid(); ApplicationDeployment applicationDeployment = new ApplicationDeployment( cluster: GetClusterAddress(clusterAddress, clusterPort), status: ApplicationDeployStatus.Copy, imageStorePath: null, applicationTypeName: package.ApplicationTypeName, applicationTypeVersion: package.ApplicationTypeVersion, applicationInstanceName: GetApplicationInstanceName(package.PackageFileName), packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName), timestamp: DateTimeOffset.UtcNow); await dictionary.AddAsync(tx, id, applicationDeployment); await queue.EnqueueAsync(tx, id); workIds.Add(id); } await tx.CommitAsync(); return(workIds); } }
internal async Task <ApplicationDeployment> ProcessApplicationDeployment( ApplicationDeployment applicationDeployment, CancellationToken cancellationToken) { try { switch (applicationDeployment.Status) { case ApplicationDeployStatus.Copy: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Copying to image store. Cluster: {0}. Application: {1}. Package path: {2}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath); try { // Unzip the package contents to a temporary location on disk. if (!this.applicationPackageTempDirectory.Exists) { this.applicationPackageTempDirectory.Create(); } FileInfo packageFile = new FileInfo(applicationDeployment.PackageZipFilePath); DirectoryInfo packageExtractLocation = new DirectoryInfo(Path.Combine(this.applicationPackageTempDirectory.FullName, applicationDeployment.ApplicationInstanceName)); if (packageExtractLocation.Exists) { packageExtractLocation.Delete(true); } ServiceEventSource.Current.ServiceMessage( this, "Extracting application package from {0} to {1}", packageFile.FullName, packageExtractLocation.FullName); ZipFile.ExtractToDirectory(packageFile.FullName, packageExtractLocation.FullName); // Copy the unzipped application package to the cluster's imagestore and clean up. string imageStorePath = await this.applicationOperator.CopyPackageToImageStoreAsync( applicationDeployment.Cluster, packageExtractLocation.FullName, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, cancellationToken); packageExtractLocation.Delete(true); return(new ApplicationDeployment( applicationDeployment.Cluster, ApplicationDeployStatus.Register, imageStorePath, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, applicationDeployment.DeploymentTimestamp)); } catch (FabricServiceNotFoundException fsnfe) { // image store service isn't ready yet. // This is retry-able, just need to wait a bit for it to come up. // This can happen when an application deployment is attempted immediately after a cluster comes up. ServiceEventSource.Current.ApplicationDeploymentFailureCopyFailure( applicationDeployment.Cluster, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, fsnfe.Message); return(applicationDeployment); } catch (FileNotFoundException fnfe) { ServiceEventSource.Current.ApplicationDeploymentFailureCorruptPackage( applicationDeployment.Cluster, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, fnfe.Message); return(new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment)); } case ApplicationDeployStatus.Register: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Registering. Cluster: {0}. Application: {1}, Imagestore path: {2}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.ImageStorePath); try { await this.applicationOperator.RegisterApplicationAsync( applicationDeployment.Cluster, applicationDeployment.ImageStorePath, cancellationToken); return(new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment)); } catch (FabricElementAlreadyExistsException) { ServiceEventSource.Current.ApplicationDeploymentFailureAlreadyRegistered( applicationDeployment.Cluster, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath); // application already exists, set status to Create it so we don't keep failing on this. return(new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment)); } case ApplicationDeployStatus.Create: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Creating. Cluster: {0}. Application: {1}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName); await this.applicationOperator.CreateApplicationAsync( applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, cancellationToken); return(new ApplicationDeployment(ApplicationDeployStatus.Complete, applicationDeployment)); default: return(applicationDeployment); } } catch (FabricTransientException fte) { ServiceEventSource.Current.ApplicationDeploymentFailureTransientError( applicationDeployment.Cluster, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, Enum.GetName(typeof(ApplicationDeployStatus), applicationDeployment.Status), fte.Message); // return the deployment unchanged. It will be returned to the queue and tried again. return(applicationDeployment); } catch (Exception e) { ServiceEventSource.Current.ApplicationDeploymentFailureUnknownError( applicationDeployment.Cluster, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, e.ToString()); return(new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment)); } }
/// <summary> /// Queues deployment of application packages to the given cluster. /// </summary> /// <param name="clusterAddress"></param> /// <param name="clusterPort"></param> /// <returns></returns> public async Task<IEnumerable<Guid>> QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort) { IReliableQueue<Guid> queue = await this.StateManager.GetOrAddAsync<IReliableQueue<Guid>>(QueueName); IReliableDictionary<Guid, ApplicationDeployment> dictionary = await this.StateManager.GetOrAddAsync<IReliableDictionary<Guid, ApplicationDeployment>>(DictionaryName); List<Guid> workIds = new List<Guid>(this.ApplicationPackages.Count()); using (ITransaction tx = this.StateManager.CreateTransaction()) { // Grab each application package that's included with the service // and create an ApplicationDeployment record of it. // Then queue a job to begin processing each one. foreach (ApplicationPackageInfo package in this.ApplicationPackages) { Guid id = Guid.NewGuid(); ApplicationDeployment applicationDeployment = new ApplicationDeployment( cluster: GetClusterAddress(clusterAddress, clusterPort), status: ApplicationDeployStatus.Copy, imageStorePath: null, applicationTypeName: package.ApplicationTypeName, applicationTypeVersion: package.ApplicationTypeVersion, applicationInstanceName: GetApplicationInstanceName(package.PackageFileName), packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName), timestamp: DateTimeOffset.UtcNow); await dictionary.AddAsync(tx, id, applicationDeployment); await queue.EnqueueAsync(tx, id); workIds.Add(id); } await tx.CommitAsync(); return workIds; } }
internal async Task<ApplicationDeployment> ProcessApplicationDeployment( ApplicationDeployment applicationDeployment, CancellationToken cancellationToken) { try { switch (applicationDeployment.Status) { case ApplicationDeployStatus.Copy: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Copying to image store. Cluster: {0}. Application: {1}. Package path: {2}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath); try { // Unzip the package contents to a temporary location on disk. if (!this.applicationPackageTempDirectory.Exists) { this.applicationPackageTempDirectory.Create(); } FileInfo packageFile = new FileInfo(applicationDeployment.PackageZipFilePath); DirectoryInfo packageExtractLocation = new DirectoryInfo(Path.Combine(this.applicationPackageTempDirectory.FullName, applicationDeployment.ApplicationInstanceName)); if (packageExtractLocation.Exists) { packageExtractLocation.Delete(true); } ServiceEventSource.Current.ServiceMessage( this, "Extracting application package from {0} to {1}", packageFile.FullName, packageExtractLocation.FullName); ZipFile.ExtractToDirectory(packageFile.FullName, packageExtractLocation.FullName); // Copy the unzipped application package to the cluster's imagestore and clean up. string imageStorePath = await this.applicationOperator.CopyPackageToImageStoreAsync( applicationDeployment.Cluster, packageExtractLocation.FullName, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, cancellationToken); packageExtractLocation.Delete(true); return new ApplicationDeployment( applicationDeployment.Cluster, ApplicationDeployStatus.Register, imageStorePath, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, applicationDeployment.ApplicationInstanceName, applicationDeployment.PackageZipFilePath, applicationDeployment.DeploymentTimestamp); } catch (FabricServiceNotFoundException fsnfe) { // image store service isn't ready yet. // This is retry-able, just need to wait a bit for it to come up. // This can happen when an application deployment is attempted immediately after a cluster comes up. ServiceEventSource.Current.ServiceMessage( this, "Copy to image store failed with FabricServiceNotFoundException. Package: {0}. Error: {1}", applicationDeployment.PackageZipFilePath, fsnfe.Message); return applicationDeployment; } catch (FileNotFoundException fnfe) { ServiceEventSource.Current.ServiceMessage( this, "Found corrupt application package. Package: {0}. Error: {1}", applicationDeployment.PackageZipFilePath, fnfe.Message); return new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment); } case ApplicationDeployStatus.Register: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Registering. Cluster: {0}. Application: {1}, Imagestore path: {2}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.ImageStorePath); try { await this.applicationOperator.RegisterApplicationAsync( applicationDeployment.Cluster, applicationDeployment.ImageStorePath, cancellationToken); return new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment); } catch (FabricElementAlreadyExistsException) { ServiceEventSource.Current.ServiceMessage( this, "Application package is already registered. Package: {0}.", applicationDeployment.PackageZipFilePath); // application already exists, set status to Create it so we don't keep failing on this. return new ApplicationDeployment(ApplicationDeployStatus.Create, applicationDeployment); } case ApplicationDeployStatus.Create: ServiceEventSource.Current.ServiceMessage( this, "Application deployment: Creating. Cluster: {0}. Application: {1}", applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName); await this.applicationOperator.CreateApplicationAsync( applicationDeployment.Cluster, applicationDeployment.ApplicationInstanceName, applicationDeployment.ApplicationTypeName, applicationDeployment.ApplicationTypeVersion, cancellationToken); return new ApplicationDeployment(ApplicationDeployStatus.Complete, applicationDeployment); default: return applicationDeployment; } } catch (FabricTransientException fte) { ServiceEventSource.Current.ServiceMessage( this, "A transient error occured during package processing. Package: {0}. Stage: {1}. Error: {2}", applicationDeployment.ApplicationInstanceName, applicationDeployment.Status, fte.Message); // return the deployment unchanged. It will be returned to the queue and tried again. return applicationDeployment; } catch (Exception e) { ServiceEventSource.Current.ServiceMessage( this, "Application package processing failed. Package: {0}. Error: {1}", applicationDeployment.PackageZipFilePath, e.ToString()); return new ApplicationDeployment(ApplicationDeployStatus.Failed, applicationDeployment); } }