private void GetWuOperationResultCount(WindowsUpdateOperationResult operationResult, out int totalOperations, out int abortedOperations, out int succeededOperations)
{
totalOperations = 0;
abortedOperations = 0;
succeededOperations = 0;
if (operationResult == null)
{
return;
}
totalOperations = operationResult.UpdateDetails.Count;
foreach (var updateDetail in operationResult.UpdateDetails)
{
switch (updateDetail.ResultCode)
{
case WuOperationResult.Aborted:
abortedOperations++;
break;
case WuOperationResult.Succeeded:
succeededOperations++;
break;
}
}
}
/// <summary>
/// Processes arguments recieved from commandline
/// </summary>
/// <param name="args">Array of arguments</param>
/// <para>Arguments specified in the commandline are case sensitive, following commands are supported
/// Format -- CommandName list_of_arguments
/// Eg: Command argument1 argument2 argument3, Following are the allowed commands and their template
/// GetWuOperationState nodeName TimeoutInSeconds
/// UpdateSearchAndDownloadStatus nodeName applicationName WuOperationStates InstallationTimeoutInMinutes TimeoutInSeconds ResultFilePath
/// UpdateInstallationStatus nodeName applicationName WuOperationStates TimeoutInSeconds ResultFilePath
/// ReportHealth applicationName title description HealthState TimeToLiveInMinutes TimeoutInSeconds
/// GetApplicationDeployedStatus applicationUri TimeoutInSeconds
/// </para>
public async Task <NodeAgentSfUtilityExitCodes> ProcessArguments(String[] args)
{
if (!args.Any())
{
return(NodeAgentSfUtilityExitCodes.InvalidArgument);
}
switch (args[0])
{
case "GetWuOperationState":
return(await this.GetWuOperationStateAsync(args[1], TimeSpan.FromSeconds(int.Parse(args[2])), this.cancellationTokenSource.Token));
case "UpdateSearchAndDownloadStatus":
return
(await
this.UpdateSearchAndDownloadStatusAsync(
args[1],
new Uri(args[2]),
(NodeAgentSfUtilityExitCodes)Enum.Parse(typeof(NodeAgentSfUtilityExitCodes), (args[3])),
(args.Count() == 7)?WindowsUpdateOperationResult.Deserialize(args[6]) : null,
int.Parse(args[4]),
new TimeSpan(0, 0, int.Parse(args[5])),
this.cancellationTokenSource.Token));
case "UpdateInstallationStatus":
return
(await
this.UpdateInstallationStatusAsync(
args[1],
new Uri(args[2]),
(NodeAgentSfUtilityExitCodes)Enum.Parse(typeof(NodeAgentSfUtilityExitCodes), (args[3])),
(args.Count() == 6)?WindowsUpdateOperationResult.Deserialize(args[5]) : null,
new TimeSpan(0, 0, int.Parse(args[4])),
this.cancellationTokenSource.Token));
case "ReportHealth":
return
(this.ReportHealth(
new Uri(args[1]),
args[2],
args[3],
(HealthState)Enum.Parse(typeof(HealthState), (args[4])),
long.Parse(args[5]),
TimeSpan.FromSeconds(int.Parse(args[6])),
this.cancellationTokenSource.Token));
case "GetApplicationDeployedStatus":
return
(await this.GetApplicationDeployedStatusAsync(new Uri(args[1]), TimeSpan.FromSeconds(int.Parse(args[2])),
this.cancellationTokenSource.Token));
default:
{
string errorMessage = String.Format("Unknown command = {0} recieved", args[0]);
ServiceEventSource.Current.ErrorMessage(errorMessage);
return(NodeAgentSfUtilityExitCodes.InvalidArgument);
}
}
}
/// <summary>
/// Updates the ResultStore of Coordinator Service with operation result
/// </summary>
/// <param name="operationResult">Result for the Windows Update operation <see cref="WindowsUpdateOperationResult"/></param>
/// <param name="timeout">timeout for the operation</param>
/// <param name="cancellationToken">cancellation token for the async task</param>
/// <returns>Task for the async operation</returns>
public async Task UpdateWuOperationResult(WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken)
{
ServiceEventSource.Current.InfoMessage(
"UpdateWuOperationResult recieved {0} {1} results from node = {2}, Operation time = {3}",
operationResult.UpdateDetails.Count,
operationResult.OperationType,
operationResult.NodeName,
operationResult.OperationTime);
// Update the result in ResultStore
try
{
var resultsStore = await this.StateManager.GetOrAddAsync <IReliableQueue <WindowsUpdateOperationResult> >(ResultsStoreName);
using (var tx = this.StateManager.CreateTransaction())
{
await resultsStore.EnqueueAsync(tx, operationResult, timeout, cancellationToken);
await tx.CommitAsync();
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ErrorMessage("Storing of result failed with exception = {0}", ex);
throw;
}
// Send the telemetry event
try
{
if (operationResult.OperationType == WindowsUpdateOperationType.Installation)
{
this.telemetryEvents.PatchInstallationEvent(
operationResult.NodeName,
operationResult.WindowsUpdateFrequency,
operationResult.WindowsUpdateQuery,
this.rmHelper.RmPolicy.ToString(),
operationResult.UpdateDetails.Count,
operationResult.UpdateDetails.Count(update => update.ResultCode == WuOperationResult.Succeeded),
operationResult.OperationResult.ToString());
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ErrorMessage("Telemetry failure ex = {0}", ex);
// Do not re-throw, telemetry failure is not a critical error
}
}
/// <summary>
/// Posts latest operation status of the current node in the form of health report
/// </summary>
/// <returns></returns>
public void ReportCurrentNodeStatus(WindowsUpdateOperationResult result)
{
if (result != null && result.UpdateDetails != null)
{
StringBuilder message = new StringBuilder();
message.AppendFormat("{0} updates were {1} on {2}", result.UpdateDetails.Count,
result.OperationType == WindowsUpdateOperationType.SearchAndDownload ? "downloaded" : "installed",
result.OperationTime.ToString("dddd, dd MMMM yyyy"));
foreach (var update in result.UpdateDetails)
{
message.AppendFormat("\nUpdateTitle : {0} , Result = {1}", update.Title, Enum.GetName(typeof(WuOperationResult), update.ResultCode));
}
message.AppendFormat("\nFor detailed results refer to https://docs.microsoft.com/azure/service-fabric/service-fabric-patch-orchestration-application#view-the-windows-update-results");
this.ReportHealth("WindowsUpdateStatus", message.ToString(), HealthState.Ok, -1,
TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes));
}
}
/// <summary>
/// Updates Windows Update Operation Result
/// </summary>
/// <param name="fabricClient">fabric client to carry out operations on service fabric</param>
/// <param name="applicationUri">Application name of the Patch Orchestration Application</param>
/// <param name="operationResult">Result of Windows Update operation <see cref="WindowsUpdateOperationResult"/></param>
/// <param name="timeout">Timeout for the operation</param>
/// <param name="cancellationToken">Cancellation token to cancel this async operation</param>
/// <returns>A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes"/></returns>
internal static async Task <NodeAgentSfUtilityExitCodes> UpdateWuOperationResult(
FabricClient fabricClient,
Uri applicationUri,
WindowsUpdateOperationResult operationResult,
TimeSpan timeout,
CancellationToken cancellationToken)
{
try
{
IDataInterface coordinatorServiceDataClient = ServiceProxy.Create <IDataInterface>(new Uri(applicationUri + CoordinatorServiceSuffix));
await
coordinatorServiceDataClient.UpdateWuOperationResult(
operationResult,
timeout,
cancellationToken);
return(NodeAgentSfUtilityExitCodes.Success);
}
catch (Exception e)
{
ServiceEventSource.Current.ErrorMessage(
String.Format("CoordinatorServiceHelper.UpdateWuOperationResult failed. Exception details {0}", e));
if (e is FabricTransientException)
{
return(NodeAgentSfUtilityExitCodes.RetryableException);
}
else if (e is FabricServiceNotFoundException)
{
return(NodeAgentSfUtilityExitCodes.ServiceNotFound);
}
else
{
return(NodeAgentSfUtilityExitCodes.Failure);
}
}
}
private bool HandleWUOperationStates(CancellationToken cancellationToken)
{
TimeSpan utilityTaskTimeOut = TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes);
NodeAgentSfUtilityExitCodes wuOperationState = this._nodeAgentSfUtility.GetWuOperationState(utilityTaskTimeOut);
_eventSource.InfoMessage("Current WU Operation State : {0}", wuOperationState);
bool reschedule = false;
switch (wuOperationState)
{
case NodeAgentSfUtilityExitCodes.None:
case NodeAgentSfUtilityExitCodes.OperationCompleted:
{
OperationResultCode searchResult = SearchUpdates(cancellationToken);
reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule);
if (this._wuCollectionWrapper != null)
{
if (this._wuCollectionWrapper.Collection.Count == 0)
{
_eventSource.InfoMessage("No Windows Update available. Completing the operation.");
//Complete operation.
this._nodeAgentSfUtility.UpdateSearchAndDownloadStatus(
NodeAgentSfUtilityExitCodes.OperationCompleted,
this._operationResultFormatter.CreateSearchAndDownloadDummyResult(),
utilityTaskTimeOut
);
break;
}
OperationResultCode downloadResult = DownloadUpdates(cancellationToken);
reschedule = (downloadResult != OperationResultCode.orcSucceeded ? true : reschedule);
WindowsUpdateOperationResult searchAndDownloadResult = this._operationResultFormatter.FormatSearchAndDownloadResult(downloadResult, this._wuCollectionWrapper);
_eventSource.InfoMessage("Search and download result: {0}", searchAndDownloadResult);
this._nodeAgentSfUtility.UpdateSearchAndDownloadStatus(NodeAgentSfUtilityExitCodes.DownloadCompleted, searchAndDownloadResult, utilityTaskTimeOut);
NodeAgentSfUtilityExitCodes exitCodes = this.WaitForInstallationApproval(cancellationToken);
if (exitCodes.Equals(NodeAgentSfUtilityExitCodes.Failure))
{
_eventSource.ErrorMessage("Not able to move from DownloadCompleted state to InstallationApproved state.");
reschedule = true;
break;
}
this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationInProgress, null, utilityTaskTimeOut);
OperationResultCode installResult = InstallUpdates(cancellationToken);
reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule);
WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper);
_eventSource.InfoMessage("Installation result: {0}", installationResult);
this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut);
}
break;
}
case NodeAgentSfUtilityExitCodes.DownloadCompleted:
case NodeAgentSfUtilityExitCodes.InstallationApproved:
{
NodeAgentSfUtilityExitCodes exitCodes = this.WaitForInstallationApproval(cancellationToken);
if (exitCodes.Equals(NodeAgentSfUtilityExitCodes.Failure))
{
_eventSource.ErrorMessage("Not able to move from DownloadCompleted state to InstallationApproved state.");
reschedule = true;
break;
}
OperationResultCode searchResult = SearchUpdates(cancellationToken);
reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule);
if (this._wuCollectionWrapper != null)
{
if (this._wuCollectionWrapper.Collection.Count == 0)
{
string msg =
"Current Operation state : InstallationApproved but no updates found to install. Completing the operation.";
this._nodeAgentSfUtility.ReportHealth("WindowsUpdateOperationResult", msg, HealthState.Warning, -1, TimeSpan.FromMinutes(this._serviceSettings.OperationTimeOutInMinutes));
_eventSource.WarningMessage(msg);
//Complete operation.
this._nodeAgentSfUtility.UpdateInstallationStatus(
NodeAgentSfUtilityExitCodes.OperationCompleted,
this._operationResultFormatter.CreateInstallationDummyResult(),
utilityTaskTimeOut
);
break;
}
this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationInProgress, null, utilityTaskTimeOut);
OperationResultCode installResult = InstallUpdates(cancellationToken);
reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule);
WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper);
_eventSource.InfoMessage("Installation result: {0}", installationResult);
this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut);
}
break;
}
case NodeAgentSfUtilityExitCodes.InstallationInProgress:
{
OperationResultCode searchResult = SearchUpdates(cancellationToken);
reschedule = (searchResult != OperationResultCode.orcSucceeded ? true : reschedule);
if (this._wuCollectionWrapper != null)
{
if (this._wuCollectionWrapper.Collection.Count == 0)
{
//this is possible when installation is completed but NT service is killed before updating "InstallationCompleted" status.
break;
}
OperationResultCode installResult = InstallUpdates(cancellationToken);
reschedule = (installResult != OperationResultCode.orcSucceeded ? true : reschedule);
WindowsUpdateOperationResult installationResult = this._operationResultFormatter.FormatInstallationResult(installResult, this._wuCollectionWrapper);
_eventSource.InfoMessage("Installation result: {0}", installationResult);
this._nodeAgentSfUtility.UpdateInstallationStatus(NodeAgentSfUtilityExitCodes.InstallationCompleted, installationResult, utilityTaskTimeOut);
}
break;
}
default:
break;
}
return(reschedule);
}
/// <summary>
/// Updates the status of installation operation
/// </summary>
/// <param name="updateState">State of Wu operation, possible values are DownloadAvailable, DownloadCompleted, OperationCompleted</param>
/// <param name="operationResult">result of the search and download operation, can be null in case no results are there to be updated</param>
/// <returns>true if operation is success else false.</returns>
public bool UpdateInstallationStatus(NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult = null, TimeSpan timeout = default(TimeSpan))
{
_eventSource.InfoMessage("Updating installation status : updateState : {0}, operationResult : {1}, timeout : {2}", updateState, operationResult, timeout);
string filePath = Path.Combine(this._settingsManager.TempFolder, OperationResultFileName);
string[] arguments;
if (operationResult != null)
{
operationResult.Serialize(filePath);
arguments = new string[] { "UpdateInstallationStatus", this._nodeName, this._applicationUri.ToString(), updateState.ToString(), timeout.TotalSeconds.ToString(), filePath };
}
else
{
arguments = new string[] { "UpdateInstallationStatus", this._nodeName, this._applicationUri.ToString(), updateState.ToString(), timeout.TotalSeconds.ToString() };
}
ProcessExecutor processExecutor = new ProcessExecutor(SfUtilityFileName, CreateProcessArgument(arguments));
long retries = 0;
while (!this._cancellationToken.IsCancellationRequested)
{
if (processExecutor.Execute() == 0)
{
return(true);
}
if (retries >= this._serviceSettings.WUOperationRetryCount || this._cancellationToken.IsCancellationRequested)
{
break;
}
TimeSpan retryDelayTime = TimeSpan.FromMinutes(this._serviceSettings.WUDelayBetweenRetriesInMinutes);
this._helper.WaitOnTask(Task.Delay(retryDelayTime), this._cancellationToken);
retries++;
}
throw new Exception("Not able to update installation status.");
}
public Task <NodeAgentSfUtilityExitCodes> UpdateInstallationStatusAsync(string nodeName, Uri applicationName, NodeAgentSfUtilityExitCodes updateState,
WindowsUpdateOperationResult operationResult, TimeSpan timeout, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
/// <summary>
/// Update the status of installation operation in Coordinator Service's Relaible store
/// Also update the status of repair task as per the updateState provided
/// </summary>
/// <param name="nodeName">Name of the service fabric node</param>
/// <param name="applicationName">Uri of the Patch Orchestration Application.</param>
/// <param name="updateState">State of Wu operation, possible values are InstallationInProgress, InstallationCompleted, RestartRequested, RestartNotNeeded, OperationCompleted</param>
/// <param name="operationResult">result of the install operation, can be null in case no results are there to be updated</param>
/// <param name="timeout">Timeout for the async operation</param>
/// <param name="cancellationToken">The cancellation token to cancel the async operation</param>
/// <returns>
/// A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes.Success"/> in case of success
/// Any other <see cref="NodeAgentSfUtilityExitCodes"/> in case of failure
/// </returns>
public async Task <NodeAgentSfUtilityExitCodes> UpdateInstallationStatusAsync(String nodeName, Uri applicationName,
NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult,
TimeSpan timeout, CancellationToken cancellationToken)
{
NodeAgentSfUtilityExitCodes result;
String resultDetails = null;
if (operationResult != null)
{
int succeededOperations;
int abortedOperations;
int totalOperations;
this.GetWuOperationResultCount(operationResult, out totalOperations, out abortedOperations, out succeededOperations);
result =
await
CoordinatorServiceHelper.UpdateWuOperationResult(
this.fabricClient,
applicationName,
operationResult,
timeout,
cancellationToken);
if (result != NodeAgentSfUtilityExitCodes.Success)
{
return(result);
}
resultDetails =
String.Format(
"{0} out of {1} updates were installed successfully, {2} were aborted",
succeededOperations,
operationResult.UpdateDetails.Count,
abortedOperations);
}
switch (updateState)
{
case NodeAgentSfUtilityExitCodes.InstallationCompleted:
{
result =
await
RepairManagerHelper.UpdateRepairTask(
this.fabricClient,
nodeName,
RepairTaskState.Executing,
RepairTaskResult.Pending,
resultDetails,
NodeAgentSfUtilityExitCodes.InstallationCompleted,
timeout,
cancellationToken);
break;
}
case NodeAgentSfUtilityExitCodes.InstallationInProgress:
{
result = await
RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Executing, RepairTaskResult.Pending, resultDetails, NodeAgentSfUtilityExitCodes.InstallationInProgress, timeout,
cancellationToken);
break;
}
case NodeAgentSfUtilityExitCodes.RestartRequested:
{
resultDetails = "Installation of the updates completed, Restart pending";
result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Executing, RepairTaskResult.Pending,
resultDetails, NodeAgentSfUtilityExitCodes.RestartRequested, timeout, cancellationToken, DateTime.UtcNow);
break;
}
case NodeAgentSfUtilityExitCodes.RestartNotNeeded:
{
resultDetails = "Installation of the updates completed, Restart not needed";
result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Executing, RepairTaskResult.Pending,
resultDetails, NodeAgentSfUtilityExitCodes.RestartNotNeeded, timeout, cancellationToken);
break;
}
case NodeAgentSfUtilityExitCodes.RestartCompleted:
{
resultDetails = "Installation of the updates completed, Restart post installation completed successfully";
result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Executing, RepairTaskResult.Pending,
resultDetails, NodeAgentSfUtilityExitCodes.RestartCompleted, timeout, cancellationToken);
break;
}
case NodeAgentSfUtilityExitCodes.OperationCompleted:
{
ServiceEventSource.Current.InfoMessage(String.Format("Mark the operation as completed"));
result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Restoring, RepairTaskResult.Succeeded,
resultDetails, NodeAgentSfUtilityExitCodes.OperationCompleted, timeout, cancellationToken);
break;
}
default:
{
ServiceEventSource.Current.ErrorMessage(String.Format("UpdateInstallationStatusAsync called with invalid state {0}", updateState));
result = NodeAgentSfUtilityExitCodes.InvalidArgument;
break;
}
}
ServiceEventSource.Current.InfoMessage("UpdateInstallationStatusAsync result = {0}", result);
return(result);
}
/// <summary>
/// Updates the status of search and download operation in CoordinatorService's repliable store
/// And updates the RepairTask with appropriate state as per the updateState provided
/// </summary>
/// <param name="nodeName">Name of current service fabric node</param>
/// <param name="applicationName">Uri of the Patch Orchestration Application.</param>
/// <param name="updateState">State of Wu operation, possible values are DownloadAvailable, DownloadCompleted, OperationCompleted</param>
/// <param name="operationResult">result of the search and download operation, cannot be null</param>
/// <param name="installationTimeout">Amount of time a node can undergo installation, during installation node would be in disabled state</param>
/// <param name="timeout">Timeout for the async operation</param>
/// <param name="cancellationToken">The cancellation token to cancel the async operation</param>
/// <returns>Task containing result of operation, true for success, false for failure</returns>
/// <returns>
/// A Task representing the asnyc operation, result of the task would be <see cref="NodeAgentSfUtilityExitCodes.Success"/> in case of success
/// Any other <see cref="NodeAgentSfUtilityExitCodes"/> in case of failure
/// </returns>
public async Task <NodeAgentSfUtilityExitCodes> UpdateSearchAndDownloadStatusAsync(String nodeName, Uri applicationName,
NodeAgentSfUtilityExitCodes updateState, WindowsUpdateOperationResult operationResult, int installationTimeout, TimeSpan timeout, CancellationToken cancellationToken)
{
String taskDescription = null;
String resultDetails = null;
NodeAgentSfUtilityExitCodes result;
ExecutorDataForRmTask executorData = new ExecutorDataForRmTask()
{
ExecutorSubState = updateState,
ExecutorTimeoutInMinutes = installationTimeout
};
if (null != operationResult)
{
int succeededOperations;
int abortedOperations;
int totalOperations;
this.GetWuOperationResultCount(operationResult, out totalOperations, out abortedOperations,
out succeededOperations);
result = await
CoordinatorServiceHelper.UpdateWuOperationResult(
this.fabricClient,
applicationName,
operationResult,
timeout,
cancellationToken);
if (result != NodeAgentSfUtilityExitCodes.Success)
{
return(result);
}
taskDescription =
String.Format(
"{0} updates successfully downloaded on {1}. Creating this repair task to install the downloaded updates",
succeededOperations, operationResult.OperationTime);
resultDetails =
String.Format("{0} updates searched. {1} downloaded successfully, {2} downloads were aborted",
operationResult.UpdateDetails.Count, succeededOperations, abortedOperations);
}
switch (updateState)
{
case NodeAgentSfUtilityExitCodes.DownloadCompleted:
{
result = await
RepairManagerHelper.CreateRepairTaskForNode(
this.fabricClient,
nodeName,
taskDescription,
resultDetails,
executorData,
timeout,
cancellationToken);
break;
}
case NodeAgentSfUtilityExitCodes.OperationCompleted:
{
result = NodeAgentSfUtilityExitCodes.Success;
break;
}
case NodeAgentSfUtilityExitCodes.OperationAborted:
{
ServiceEventSource.Current.InfoMessage(String.Format("Operation aborted for a claimed task"));
result = await RepairManagerHelper.UpdateRepairTask(this.fabricClient, nodeName,
RepairTaskState.Completed, RepairTaskResult.Failed,
"Aborting the operation", updateState, timeout, cancellationToken);
break;
}
default:
{
ServiceEventSource.Current.ErrorMessage(
String.Format("UpdateSearchAndDownloadStatusAsync called with invalid state {0}", updateState));
result = NodeAgentSfUtilityExitCodes.InvalidArgument;
break;
}
}
ServiceEventSource.Current.InfoMessage("UpdateSearchAndDownloadStatusAsync result = {0}", result);
return(result);
}
