public async Task <ApiResult> RestatAllWithRetries([FromBody] JObject request)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(request, "request");
var jobNames = request.Value <JArray>("JobNames")?.Value <string[]>() ?? Array.Empty <string>();
var maxRetryTimes = request.GetValue("MaxRetryTimes", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 2;
var waitIntervalInSeconds = request.GetValue("WaitIntervalInSeconds", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 2;
var maxWaitTimeInSeconds = request.GetValue("MaxWaitTimeInSeconds", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 60;
int retryTimes = 0;
IEnumerable <SparkJobFrontEnd> notReadyJobs = Enumerable.Empty <SparkJobFrontEnd>();
do
{
await _jobOperation.RestartAllJobs(jobNames);
var startTime = DateTime.Now;
do
{
SparkJobFrontEnd[] jobOpResult = jobNames != null && jobNames.Any() ?
await _jobOperation.SyncJobStateByNames(jobNames)
: await _jobOperation.SyncAllJobState();
notReadyJobs = jobOpResult.Where(r => r.JobState == JobState.Starting || r.JobState == JobState.Idle || r.JobState == JobState.Error);
jobNames = notReadyJobs.Select(j => j.Name).ToArray();
if (!notReadyJobs.Any())
{
break;
}
System.Threading.Thread.Sleep(waitIntervalInSeconds * 1000);
} while ((DateTime.Now - startTime).TotalSeconds < maxWaitTimeInSeconds);
} while (retryTimes++ < maxRetryTimes && notReadyJobs.Any());
return(ApiResult.CreateSuccess(JToken.FromObject(notReadyJobs)));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
public async Task <ApiResult> SyncAllJobs()
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
// Sync all jobs
var jobOpResult = await _jobOperation.SyncAllJobState();
return(ApiResult.CreateSuccess(JToken.FromObject(jobOpResult)));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
