private void GetWuOperationResultCount(WindowsUpdateOperationResult operationResult, out int totalOperations, out int abortedOperations, out int succeededOperations) { totalOperations = 0; abortedOperations = 0; succeededOperations = 0; if (operationResult == null) { return; } totalOperations = operationResult.UpdateDetails.Count; foreach (var updateDetail in operationResult.UpdateDetails) { switch (updateDetail.ResultCode) { case WuOperationResult.Aborted: abortedOperations++; break; case WuOperationResult.Succeeded: succeededOperations++; break; } } }
/// <summary> /// Processes arguments recieved from commandline /// </summary> /// <param name="args">Array of arguments</param> /// <para>Arguments specified in the commandline are case sensitive, following commands are supported /// Format -- CommandName list_of_arguments /// Eg: Command argument1 argument2 argument3, Following are the allowed commands and their template /// GetWuOperationState nodeName TimeoutInSeconds /// UpdateSearchAndDownloadStatus nodeName applicationName WuOperationStates InstallationTimeoutInMinutes TimeoutInSeconds ResultFilePath /// UpdateInstallationStatus nodeName applicationName WuOperationStates TimeoutInSeconds ResultFilePath /// ReportHealth applicationName title description HealthState TimeToLiveInMinutes TimeoutInSeconds /// GetApplicationDeployedStatus applicationUri TimeoutInSeconds /// </para> public async Task <NodeAgentSfUtilityExitCodes> ProcessArguments(String[] args) { if (!args.Any()) { return(NodeAgentSfUtilityExitCodes.InvalidArgument); } switch (args[0]) { case "GetWuOperationState": return(await this.GetWuOperationStateAsync(args[1], TimeSpan.FromSeconds(int.Parse(args[2])), this.cancellationTokenSource.Token)); case "UpdateSearchAndDownloadStatus": return (await this.UpdateSearchAndDownloadStatusAsync( args[1], new Uri(args[2]), (NodeAgentSfUtilityExitCodes)Enum.Parse(typeof(NodeAgentSfUtilityExitCodes), (args[3])), (args.Count() == 7)?WindowsUpdateOperationResult.Deserialize(args[6]) : null, int.Parse(args[4]), new TimeSpan(0, 0, int.Parse(args[5])), this.cancellationTokenSource.Token)); case "UpdateInstallationStatus": return (await this.UpdateInstallationStatusAsync( args[1], new Uri(args[2]), (NodeAgentSfUtilityExitCodes)Enum.Parse(typeof(NodeAgentSfUtilityExitCodes), (args[3])), (args.Count() == 6)?WindowsUpdateOperationResult.Deserialize(args[5]) : null, new TimeSpan(0, 0, int.Parse(args[4])), this.cancellationTokenSource.Token)); case "ReportHealth": return (this.ReportHealth( new Uri(args[1]), args[2], args[3], (HealthState)Enum.Parse(typeof(HealthState), (args[4])), long.Parse(args[5]), TimeSpan.FromSeconds(int.Parse(args[6])), this.cancellationTokenSource.Token)); case "GetApplicationDeployedStatus": return (await this.GetApplicationDeployedStatusAsync(new Uri(args[1]), TimeSpan.FromSeconds(int.Parse(args[2])), this.cancellationTokenSource.Token)); default: { string errorMessage = String.Format("Unknown command = {0} recieved", args[0]); ServiceEventSource.Current.ErrorMessage(errorMessage); return(NodeAgentSfUtilityExitCodes.InvalidArgument); } } }
/// <summary> /// Updates the ResultStore of Coordinator Service with operation result /// </summary> /// <param name="operationResult">Result for the Windows Update operation <see cref="WindowsUpdateOperationResult"/></param> /// <param name="timeout">timeout for the operation</param> /// <param name="cancellationToken">cancellation token for the async task</param> /// <returns>Task for the async operation</returns> public async Task UpdateWuOperationResult(WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken) { ServiceEventSource.Current.InfoMessage( "UpdateWuOperationResult recieved {0} {1} results from node = {2}, Operation time = {3}", operationResult.UpdateDetails.Count, operationResult.OperationType, operationResult.NodeName, operationResult.OperationTime); // Update the result in ResultStore try { var resultsStore = await this.StateManager.GetOrAddAsync <IReliableQueue <WindowsUpdateOperationResult> >(ResultsStoreName); using (var tx = this.StateManager.CreateTransaction()) { await resultsStore.EnqueueAsync(tx, operationResult, timeout, cancellationToken); await tx.CommitAsync(); } } catch (Exception ex) { ServiceEventSource.Current.ErrorMessage("Storing of result failed with exception = {0}", ex); throw; } // Send the telemetry event try { if (operationResult.OperationType == WindowsUpdateOperationType.Installation) { this.telemetryEvents.PatchInstallationEvent( operationResult.NodeName, operationResult.WindowsUpdateFrequency, operationResult.WindowsUpdateQuery, this.rmHelper.RmPolicy.ToString(), operationResult.UpdateDetails.Count, operationResult.UpdateDetails.Count(update => update.ResultCode == WuOperationResult.Succeeded), operationResult.OperationResult.ToString()); } } catch (Exception ex) { ServiceEventSource.Current.ErrorMessage("Telemetry failure ex = {0}", ex); // Do not re-throw, telemetry failure is not a critical error } }
/// <summary> /// Posts latest operation status of the current node in the form of health report /// </summary> /// <returns></returns> public void ReportCurrentNodeStatus(WindowsUpdateOperationResult result) { if (result != null && result.UpdateDetails != null) { StringBuilder message = new StringBuilder(); message.AppendFormat("{0} updates were {1} on {2}", result.UpdateDetails.Count, result.OperationType == WindowsUpdateOperationType.SearchAndDownload ? "downloaded" : "installed", result.OperationTime.ToString("dddd, dd MMMM yyyy")); foreach (var update in result.UpdateDetails) { message.AppendFormat("\nUpdateTitle : {0} , Result = {1}", update.Title, Enum.GetName(typeof(WuOperationResult), update.ResultCode)); } message.AppendFormat("\nFor detailed results refer to https://docs.microsoft.com/azure/service-fabric/service-fabric-patch-orchestration-application#view-the-windows-update-results"); this.ReportHealth("WindowsUpdateStatus", message.ToString(), HealthState.Ok, -1, TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes)); } }
/// <summary> /// Updates Windows Update Operation Result /// </summary> /// <param name="fabricClient">fabric client to carry out operations on service fabric</param> /// <param name="applicationUri">Application name of the Patch Orchestration Application</param> /// <param name="operationResult">Result of Windows Update operation <see cref="WindowsUpdateOperationResult"/></param> /// <param name="timeout">Timeout for the operation</param> /// <param name="cancellationToken">Cancellation token to cancel this async operation</param> /// <returns>A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes"/></returns> internal static async Task <NodeAgentSfUtilityExitCodes> UpdateWuOperationResult( FabricClient fabricClient, Uri applicationUri, WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken) { try { IDataInterface coordinatorServiceDataClient = ServiceProxy.Create <IDataInterface>(new Uri(applicationUri + CoordinatorServiceSuffix)); await coordinatorServiceDataClient.UpdateWuOperationResult( operationResult, timeout, cancellationToken); return(NodeAgentSfUtilityExitCodes.Success); } catch (Exception e) { ServiceEventSource.Current.ErrorMessage( String.Format("CoordinatorServiceHelper.UpdateWuOperationResult failed. Exception details {0}", e)); if (e is FabricTransientException) { return(NodeAgentSfUtilityExitCodes.RetryableException); } else if (e is FabricServiceNotFoundException) { return(NodeAgentSfUtilityExitCodes.ServiceNotFound); } else { return(NodeAgentSfUtilityExitCodes.Failure); } } }
private bool HandleWUOperationStates(CancellationToken cancellationToken) { TimeSpan utilityTaskTimeOut = TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes); NodeAgentSfUtilityExitCodes wuOperationState = this._nodeAgentSfUtility.GetWuOperationState(utilityTaskTimeOut); _eventSource.InfoMessage("Current WU Operation State : {0}", wuOperationState); bool reschedule = false; switch (wuOperationState) { case NodeAgentSfUtilityExitCodes.None: case NodeAgentSfUtilityExitCodes.OperationCompleted: { OperationResultCode searchResult = SearchUpdates(cancellationToken); reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule); if (this._wuCollectionWrapper != null) { if (this._wuCollectionWrapper.Collection.Count == 0) { _eventSource.InfoMessage("No Windows Update available. Completing the operation."); //Complete operation. this._nodeAgentSfUtility.UpdateSearchAndDownloadStatus( NodeAgentSfUtilityExitCodes.OperationCompleted, this._operationResultFormatter.CreateSearchAndDownloadDummyResult(), utilityTaskTimeOut ); break; } OperationResultCode downloadResult = DownloadUpdates(cancellationToken); reschedule = (downloadResult != OperationResultCode.orcSucceeded ? true : reschedule); WindowsUpdateOperationResult searchAndDownloadResult = this._operationResultFormatter.FormatSearchAndDownloadResult(downloadResult, this._wuCollectionWrapper); _eventSource.InfoMessage("Search and download result: {0}", searchAndDownloadResult); this._nodeAgentSfUtility.UpdateSearchAndDownloadStatus(NodeAgentSfUtilityExitCodes.DownloadCompleted, searchAndDownloadResult, utilityTaskTimeOut); NodeAgentSfUtilityExitCodes exitCodes = this.WaitForInstallationApproval(cancellationToken); if (exitCodes.Equals(NodeAgentSfUtilityExitCodes.Failure)) { _eventSource.ErrorMessage("Not able to move from DownloadCompleted state to InstallationApproved state."); reschedule = true; break; } this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationInProgress, null, utilityTaskTimeOut); OperationResultCode installResult = InstallUpdates(cancellationToken); reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule); WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper); _eventSource.InfoMessage("Installation result: {0}", installationResult); this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut); } break; } case NodeAgentSfUtilityExitCodes.DownloadCompleted: case NodeAgentSfUtilityExitCodes.InstallationApproved: { NodeAgentSfUtilityExitCodes exitCodes = this.WaitForInstallationApproval(cancellationToken); if (exitCodes.Equals(NodeAgentSfUtilityExitCodes.Failure)) { _eventSource.ErrorMessage("Not able to move from DownloadCompleted state to InstallationApproved state."); reschedule = true; break; } OperationResultCode searchResult = SearchUpdates(cancellationToken); reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule); if (this._wuCollectionWrapper != null) { if (this._wuCollectionWrapper.Collection.Count == 0) { string msg = "Current Operation state : InstallationApproved but no updates found to install. Completing the operation."; this._nodeAgentSfUtility.ReportHealth("WindowsUpdateOperationResult", msg, HealthState.Warning, -1, TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes)); _eventSource.WarningMessage(msg); //Complete operation. this._nodeAgentSfUtility.UpdateInstallationStatus( NodeAgentSfUtilityExitCodes.OperationCompleted, this._operationResultFormatter.CreateInstallationDummyResult(), utilityTaskTimeOut ); break; } this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationInProgress, null, utilityTaskTimeOut); OperationResultCode installResult = InstallUpdates(cancellationToken); reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule); WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper); _eventSource.InfoMessage("Installation result: {0}", installationResult); this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut); } break; } case NodeAgentSfUtilityExitCodes.InstallationInProgress: { OperationResultCode searchResult = SearchUpdates(cancellationToken); reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule); if (this._wuCollectionWrapper != null) { if (this._wuCollectionWrapper.Collection.Count == 0) { //this is possible when installation is completed but NT service is killed before updating "InstallationCompleted" status. break; } OperationResultCode installResult = InstallUpdates(cancellationToken); reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule); WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper); _eventSource.InfoMessage("Installation result: {0}", installationResult); this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut); } break; } default: break; } return(reschedule); }
/// <summary> /// Updates the status of installation operation /// </summary> /// <param name="updateState">State of Wu operation, possible values are DownloadAvailable, DownloadCompleted, OperationCompleted</param> /// <param name="operationResult">result of the search and download operation, can be null in case no results are there to be updated</param> /// <returns>true if operation is success else false.</returns> public bool UpdateInstallationStatus(NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult = null, TimeSpan timeout = default(TimeSpan)) { _eventSource.InfoMessage("Updating installation status : updateState : {0}, operationResult : {1}, timeout : {2}", updateState, operationResult, timeout); string filePath = Path.Combine(this._settingsManager.TempFolder, OperationResultFileName); string[] arguments; if (operationResult != null) { operationResult.Serialize(filePath); arguments = new string[] { "UpdateInstallationStatus", this._nodeName, this._applicationUri.ToString(), updateState.ToString(), timeout.TotalSeconds.ToString(), filePath }; } else { arguments = new string[] { "UpdateInstallationStatus", this._nodeName, this._applicationUri.ToString(), updateState.ToString(), timeout.TotalSeconds.ToString() }; } ProcessExecutor processExecutor = new ProcessExecutor(SfUtilityFileName, CreateProcessArgument(arguments)); long retries = 0; while (!this._cancellationToken.IsCancellationRequested) { if (processExecutor.Execute() == 0) { return(true); } if (retries >= this._serviceSettings.WUOperationRetryCount || this._cancellationToken.IsCancellationRequested) { break; } TimeSpan retryDelayTime = TimeSpan.FromMinutes(this._serviceSettings.WUDelayBetweenRetriesInMinutes); this._helper.WaitOnTask(Task.Delay(retryDelayTime), this._cancellationToken); retries++; } throw new Exception("Not able to update installation status."); }
public Task <NodeAgentSfUtilityExitCodes> UpdateInstallationStatusAsync(string nodeName, Uri applicationName, NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken) { throw new NotImplementedException(); }
/// <summary> /// Update the status of installation operation in Coordinator Service's Relaible store /// Also update the status of repair task as per the updateState provided /// </summary> /// <param name="nodeName">Name of the service fabric node</param> /// <param name="applicationName">Uri of the Patch Orchestration Application.</param> /// <param name="updateState">State of Wu operation, possible values are InstallationInProgress, InstallationCompleted, RestartRequested, RestartNotNeeded, OperationCompleted</param> /// <param name="operationResult">result of the install operation, can be null in case no results are there to be updated</param> /// <param name="timeout">Timeout for the async operation</param> /// <param name="cancellationToken">The cancellation token to cancel the async operation</param> /// <returns> /// A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes.Success"/> in case of success /// Any other <see cref="NodeAgentSfUtilityExitCodes"/> in case of failure /// </returns> public async Task <NodeAgentSfUtilityExitCodes> UpdateInstallationStatusAsync(String nodeName, Uri applicationName, NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken) { NodeAgentSfUtilityExitCodes result; String resultDetails = null; if (operationResult != null) { int succeededOperations; int abortedOperations; int totalOperations; this.GetWuOperationResultCount(operationResult, out totalOperations, out abortedOperations, out succeededOperations); result = await CoordinatorServiceHelper.UpdateWuOperationResult( this.fabricClient, applicationName, operationResult, timeout, cancellationToken); if (result != NodeAgentSfUtilityExitCodes.Success) { return(result); } resultDetails = String.Format( "{0} out of {1} updates were installed successfully, {2} were aborted", succeededOperations, operationResult.UpdateDetails.Count, abortedOperations); } switch (updateState) { case NodeAgentSfUtilityExitCodes.InstallationCompleted: { result = await RepairManagerHelper.UpdateRepairTask( this.fabricClient, nodeName, RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.InstallationCompleted, timeout, cancellationToken); break; } case NodeAgentSfUtilityExitCodes.InstallationInProgress: { result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.InstallationInProgress, timeout, cancellationToken); break; } case NodeAgentSfUtilityExitCodes.RestartRequested: { resultDetails = "Installation of the updates completed, Restart pending"; result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.RestartRequested, timeout, cancellationToken, DateTime.UtcNow); break; } case NodeAgentSfUtilityExitCodes.RestartNotNeeded: { resultDetails = "Installation of the updates completed, Restart not needed"; result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.RestartNotNeeded, timeout, cancellationToken); break; } case NodeAgentSfUtilityExitCodes.RestartCompleted: { resultDetails = "Installation of the updates completed, Restart post installation completed successfully"; result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.RestartCompleted, timeout, cancellationToken); break; } case NodeAgentSfUtilityExitCodes.OperationCompleted: { ServiceEventSource.Current.InfoMessage(String.Format("Mark the operation as completed")); result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Restoring, RepairTaskResult.Succeeded, resultDetails, NodeAgentSfUtilityExitCodes.OperationCompleted, timeout, cancellationToken); break; } default: { ServiceEventSource.Current.ErrorMessage(String.Format("UpdateInstallationStatusAsync called with invalid state {0}", updateState)); result = NodeAgentSfUtilityExitCodes.InvalidArgument; break; } } ServiceEventSource.Current.InfoMessage("UpdateInstallationStatusAsync result = {0}", result); return(result); }
/// <summary> /// Updates the status of search and download operation in CoordinatorService's repliable store /// And updates the RepairTask with appropriate state as per the updateState provided /// </summary> /// <param name="nodeName">Name of current service fabric node</param> /// <param name="applicationName">Uri of the Patch Orchestration Application.</param> /// <param name="updateState">State of Wu operation, possible values are DownloadAvailable, DownloadCompleted, OperationCompleted</param> /// <param name="operationResult">result of the search and download operation, cannot be null</param> /// <param name="installationTimeout">Amount of time a node can undergo installation, during installation node would be in disabled state</param> /// <param name="timeout">Timeout for the async operation</param> /// <param name="cancellationToken">The cancellation token to cancel the async operation</param> /// <returns>Task containing result of operation, true for success, false for failure</returns> /// <returns> /// A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes.Success"/> in case of success /// Any other <see cref="NodeAgentSfUtilityExitCodes"/> in case of failure /// </returns> public async Task <NodeAgentSfUtilityExitCodes> UpdateSearchAndDownloadStatusAsync(String nodeName, Uri applicationName, NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult, int installationTimeout, TimeSpan timeout, CancellationToken cancellationToken) { String taskDescription = null; String resultDetails = null; NodeAgentSfUtilityExitCodes result; ExecutorDataForRmTask executorData = new ExecutorDataForRmTask() { ExecutorSubState = updateState, ExecutorTimeoutInMinutes = installationTimeout }; if (null != operationResult) { int succeededOperations; int abortedOperations; int totalOperations; this.GetWuOperationResultCount(operationResult, out totalOperations, out abortedOperations, out succeededOperations); result = await CoordinatorServiceHelper.UpdateWuOperationResult( this.fabricClient, applicationName, operationResult, timeout, cancellationToken); if (result != NodeAgentSfUtilityExitCodes.Success) { return(result); } taskDescription = String.Format( "{0} updates successfully downloaded on {1}. Creating this repair task to install the downloaded updates", succeededOperations, operationResult.OperationTime); resultDetails = String.Format("{0} updates searched. {1} downloaded successfully, {2} downloads were aborted", operationResult.UpdateDetails.Count, succeededOperations, abortedOperations); } switch (updateState) { case NodeAgentSfUtilityExitCodes.DownloadCompleted: { result = await RepairManagerHelper.CreateRepairTaskForNode( this.fabricClient, nodeName, taskDescription, resultDetails, executorData, timeout, cancellationToken); break; } case NodeAgentSfUtilityExitCodes.OperationCompleted: { result = NodeAgentSfUtilityExitCodes.Success; break; } case NodeAgentSfUtilityExitCodes.OperationAborted: { ServiceEventSource.Current.InfoMessage(String.Format("Operation aborted for a claimed task")); result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName, RepairTaskState.Completed, RepairTaskResult.Failed, "Aborting the operation", updateState, timeout, cancellationToken); break; } default: { ServiceEventSource.Current.ErrorMessage( String.Format("UpdateSearchAndDownloadStatusAsync called with invalid state {0}", updateState)); result = NodeAgentSfUtilityExitCodes.InvalidArgument; break; } } ServiceEventSource.Current.InfoMessage("UpdateSearchAndDownloadStatusAsync result = {0}", result); return(result); }