/// <summary>
/// Processes successfully finished job.
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request to process.</param>
/// <param name="submitProvisioningRequest">Flag to indicate if provisioning request should be submitted.</param>
/// <param name="logger">Logger to log data.</param>
/// <returns></returns>
private async Task ProcessFinishedJobAsync(JobVerificationRequestModel jobVerificationRequestModel, bool submitProvisioningRequest, ILogger logger)
{
logger.LogInformation($"JobVerificationService::ProcessFinishedJob started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
// check if stream provisioning requests needs to be submitted.
// There are two scenarios, if job was completed and there is record in local storage, there is nothing to do, since this request was submitted as part of job output status process.
// If job is completed, but status is missing in storage service, this means that EventGrid event was lost and provisioning request needs to be submitted.
if (submitProvisioningRequest)
{
var provisioningRequestResult = await this.provisioningRequestStorageService.CreateAsync(
new ProvisioningRequestModel
{
Id = Guid.NewGuid().ToString(),
ProcessedAssetMediaServiceAccountName = jobVerificationRequestModel.MediaServiceAccountName,
ProcessedAssetName = jobVerificationRequestModel.JobOutputAssetName,
StreamingLocatorName = $"streaming-{jobVerificationRequestModel.OriginalJobRequestModel.OutputAssetName}"
},
logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::ProcessFinishedJob stream provisioning request submitted for completed job: provisioningRequestResult={LogHelper.FormatObjectForLog(provisioningRequestResult)}");
}
// Need to delete completed jobs not to reach max number of jobs in Azure Media Service instance
await this.DeleteJobAsync(jobVerificationRequestModel, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::ProcessFinishedJob completed: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
/// <summary>
/// Processes job that has not been completed or failed
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request to process</param>
/// <param name="logger">Logger to log data</param>
/// <returns></returns>
private async Task ProcessStuckJob(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
logger.LogInformation($"JobVerificationService::ProcessStuckJob started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
// Job has not been completed over predefined period of time, need to submit another request to verify it in the future.
await this.SubmitVerificationRequestAsync(jobVerificationRequestModel, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::ProcessStuckJob completed: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
/// <summary>
/// Deletes job info from Azure Media Service instance.
/// </summary>
/// <param name="jobVerificationRequestModel"></param>
/// <param name="logger"></param>
/// <returns></returns>
private async Task DeleteJobAsync(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
logger.LogInformation($"JobVerificationService::DeleteJobAsync started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[jobVerificationRequestModel.MediaServiceAccountName];
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(jobVerificationRequestModel.MediaServiceAccountName, logger);
await clientInstance.Jobs.DeleteWithHttpMessagesAsync(clientConfiguration.ResourceGroup, clientConfiguration.AccountName, jobVerificationRequestModel.OriginalJobRequestModel.TransformName, jobVerificationRequestModel.JobName).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::DeleteJobAsync completed: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
/// <summary>
/// Creates new job verification request. This requests is used to verify that job was successfully completed.
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request</param>
/// <param name="verificationDelay">How far in future to run verification logic</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Stored job verification request</returns>
public async Task <JobVerificationRequestModel> CreateAsync(JobVerificationRequestModel jobVerificationRequestModel, TimeSpan verificationDelay, ILogger logger)
{
var message = JsonConvert.SerializeObject(jobVerificationRequestModel, this.settings);
// Encode message to Base64 before sending to the queue
await this.queue.SendMessageAsync(QueueServiceHelper.EncodeToBase64(message), verificationDelay).ConfigureAwait(false);
logger.LogInformation($"JobVerificationRequestStorageService::CreateAsync successfully added request to the queue: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)} verificationDelay={verificationDelay}");
return(jobVerificationRequestModel);
}
/// <summary>
/// Processes failed job.
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request to process.</param>
/// <param name="jobOutputStatusModel">Job output status request to process, it should match data in above parameter.</param>
/// <param name="logger">Logger to log data.</param>
/// <returns></returns>
private async Task ProcessFailedJob(JobVerificationRequestModel jobVerificationRequestModel, JobOutputStatusModel jobOutputStatusModel, ILogger logger)
{
logger.LogInformation($"JobVerificationService::ProcessFailedJob started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)} jobOutputStatusModel={LogHelper.FormatObjectForLog(jobOutputStatusModel)}");
// Need to delete failed jobs not to reach max number of jobs in Azure Media Service instance.
await this.DeleteJobAsync(jobVerificationRequestModel, logger).ConfigureAwait(false);
// If job has failed for system errors, it needs to be resubmitted.
if (jobOutputStatusModel.HasRetriableError)
{
await this.ResubmitJob(jobVerificationRequestModel, logger).ConfigureAwait(false);
}
else
{
// no need to resubmit job that failed for user error.
logger.LogInformation($"JobVerificationService::ProcessFailedJob submitted job failed, not a system error, skipping retry: result={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
logger.LogInformation($"JobVerificationService::ProcessFailedJob completed: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)} jobOutputStatusModel={LogHelper.FormatObjectForLog(jobOutputStatusModel)}");
}
/// <summary>
/// Submits another verification request for "stuck" job.
/// </summary>
/// <param name="jobVerificationRequestModel"></param>
/// <param name="logger"></param>
/// <returns></returns>
private async Task SubmitVerificationRequestAsync(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
// This method is called for job that has not been completed in time.
// To avoid indefinite loop, need to check if it is possible to submit another verification request.
if (jobVerificationRequestModel.RetryCount < this.maxNumberOfRetries)
{
jobVerificationRequestModel.RetryCount++;
// extend verification delay.
var verificationDelay = new TimeSpan(0, this.configService.TimeDurationInMinutesToVerifyJobStatus * jobVerificationRequestModel.RetryCount, 0);
// submit new verification request with future visibility.
var retryCount = 3;
var retryTimeOut = 1000;
// Job is submitted at this point, failing to do any calls after this point would result in reprocessing this job request and submitting duplicate one.
// It is OK to retry and ignore exception at the end. In current implementation based on Azure storage, it is very unlikely to fail in any of the below calls.
do
{
try
{
var jobVerificationResult = await this.jobVerificationRequestStorageService.CreateAsync(jobVerificationRequestModel, verificationDelay, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::SubmitVerificationRequestAsync successfully submitted jobVerificationModel: result={LogHelper.FormatObjectForLog(jobVerificationResult)}");
// no exception happened, let's break.
break;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
logger.LogError($"JobVerificationService::SubmitVerificationRequestAsync got exception calling jobVerificationRequestStorageService.CreateAsync: retryCount={retryCount} message={e.Message} jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
retryCount--;
await Task.Delay(retryTimeOut).ConfigureAwait(false);
}
}while (retryCount > 0);
}
else
{
logger.LogWarning($"JobVerificationService::SubmitVerificationRequestAsync max number of retries reached to check stuck job, this job request will not be processed, please manually check if job needs to be resubmitted, skipping request: result={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
}
/// <summary>
/// Verifies the status of given job, implements business logic to resubmit jobs if needed
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Processed job verification request</returns>
public async Task <JobVerificationRequestModel> VerifyJobAsync(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
logger.LogInformation($"JobVerificationService::VerifyJobAsync started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
// Get latest job output status from storage service.
var jobOutputStatus = await this.jobOutputStatusStorageService.GetLatestJobOutputStatusAsync(jobVerificationRequestModel.JobName, jobVerificationRequestModel.JobOutputAssetName).ConfigureAwait(false);
var jobOutputStatusLoadedFromAPI = false;
// if job has not reached final state, need to reload status from Azure Media Service APIs in case of delayed or lost EventGrid event.
if (jobOutputStatus?.JobOutputState != JobState.Finished && jobOutputStatus?.JobOutputState != JobState.Error && jobOutputStatus?.JobOutputState != JobState.Canceled)
{
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[jobVerificationRequestModel.MediaServiceAccountName];
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(jobVerificationRequestModel.MediaServiceAccountName, logger);
logger.LogInformation($"JobVerificationService::VerifyJobAsync checking job status using API: mediaServiceInstanceName={jobVerificationRequestModel.MediaServiceAccountName}");
// Get job data to verify status of specific job output.
var job = await clientInstance.Jobs.GetAsync(clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobVerificationRequestModel.OriginalJobRequestModel.TransformName,
jobVerificationRequestModel.JobName).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync loaded job data from API: job={LogHelper.FormatObjectForLog(job)}");
if (job != null)
{
// create job output status record using job loaded from Azure Media Service API.
var statusInfo = MediaServicesHelper.GetJobOutputState(job, jobVerificationRequestModel.JobOutputAssetName);
jobOutputStatus = new JobOutputStatusModel
{
Id = Guid.NewGuid().ToString(),
EventTime = statusInfo.Item2,
JobOutputState = statusInfo.Item1,
JobName = job.Name,
MediaServiceAccountName = jobVerificationRequestModel.MediaServiceAccountName,
JobOutputAssetName = jobVerificationRequestModel.JobOutputAssetName,
TransformName = jobVerificationRequestModel.OriginalJobRequestModel.TransformName,
HasRetriableError = MediaServicesHelper.HasRetriableError(job, jobVerificationRequestModel.JobOutputAssetName) // check if job should be retried
};
jobOutputStatusLoadedFromAPI = true;
// persist job output status record
await this.jobOutputStatusStorageService.CreateOrUpdateAsync(jobOutputStatus, logger).ConfigureAwait(false);
}
}
// At this point here, jobOutputStatus is either loaded from job output status storage or from Azure Media Service API.
logger.LogInformation($"JobVerificationService::VerifyJobAsync jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
// Check if job output has been successfully finished.
if (jobOutputStatus?.JobOutputState == JobState.Finished)
{
await this.ProcessFinishedJobAsync(jobVerificationRequestModel, jobOutputStatusLoadedFromAPI, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job was completed successfully: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// Check if job output failed.
if (jobOutputStatus?.JobOutputState == JobState.Error)
{
await this.ProcessFailedJob(jobVerificationRequestModel, jobOutputStatus, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job failed: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// check if job has been canceled.
if (jobOutputStatus?.JobOutputState == JobState.Canceled)
{
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job canceled: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// At this point, job is stuck, it is not in the final state and long enough time period has passed (since this code is running for a given job).
await this.ProcessStuckJob(jobVerificationRequestModel, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync completed: job={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
return(jobVerificationRequestModel);
}
/// <summary>
/// Resubmits failed job.
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request to process</param>
/// <param name="logger">Logger to log data</param>
/// <returns></returns>
private async Task ResubmitJob(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
// This method is called for failed job.
// To avoid indefinite loop, need to check if it is possible to resubmit job
if (jobVerificationRequestModel.RetryCount < this.maxNumberOfRetries)
{
var selectedInstanceName = await this.mediaServiceInstanceHealthService.GetNextAvailableInstanceAsync(logger).ConfigureAwait(false);
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[selectedInstanceName];
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(selectedInstanceName, logger);
jobVerificationRequestModel.RetryCount++;
var transform = await clientInstance.Transforms.GetAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobVerificationRequestModel.OriginalJobRequestModel.TransformName).ConfigureAwait(false);
// Need to check transform output on error setting. If all outputs have continue job, no need to resubmit such job, since failure is not critical.
if (transform.Outputs.All(t => t.OnError == OnErrorType.ContinueJob))
{
logger.LogInformation($"JobVerificationService::ResubmitJob skipping request to resubmit since all transforms outputs are set to continue job: transform={LogHelper.FormatObjectForLog(transform)}");
return;
}
// Logic below is similar to JobSchedulingService implementation when initial job is submitted.
// This logic below may need to be updated if multiple job outputs are used per single job and partial resubmit is required to process only failed job outputs.
// Update output asset.
var outputAsset = await clientInstance.Assets.CreateOrUpdateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobVerificationRequestModel.JobOutputAssetName,
new Asset()).ConfigureAwait(false);
JobOutput[] jobOutputs = { new JobOutputAsset(outputAsset.Name) };
// Old job is deleted, new job can be submitted again with the same name.
var job = await clientInstance.Jobs.CreateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobVerificationRequestModel.OriginalJobRequestModel.TransformName,
jobVerificationRequestModel.JobName,
new Job
{
Input = jobVerificationRequestModel.OriginalJobRequestModel.JobInputs,
Outputs = jobOutputs,
}).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::ResubmitJob successfully re-submitted job: job={LogHelper.FormatObjectForLog(job)}");
jobVerificationRequestModel.JobId = job.Id;
jobVerificationRequestModel.MediaServiceAccountName = selectedInstanceName;
jobVerificationRequestModel.JobName = job.Name;
jobVerificationRequestModel.JobOutputAssetName = outputAsset.Name;
// new verification request is submitted to verify the result of this job in future.
await this.SubmitVerificationRequestAsync(jobVerificationRequestModel, logger).ConfigureAwait(false);
this.mediaServiceInstanceHealthService.RecordInstanceUsage(selectedInstanceName, logger);
}
else
{
logger.LogWarning($"JobVerificationService::ResubmitJob max number of retries reached to check stuck job, this job request will not be processed, please manually check if job needs to be resubmitted, skipping request: result={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
}
}
/// <summary>
/// Submits job to Azure Media Services.
/// </summary>
/// <param name="jobRequestModel">Job to submit.</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Submitted job</returns>
public async Task <Job> SubmitJobAsync(JobRequestModel jobRequestModel, ILogger logger)
{
logger.LogInformation($"JobSchedulingService::SubmitJobAsync started: jobRequestModel={LogHelper.FormatObjectForLog(jobRequestModel)}");
// Get next available Azure Media Services instance
var selectedInstanceName = await this.mediaServiceInstanceHealthService.GetNextAvailableInstanceAsync(logger).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync selected healthy instance: MediaServiceAccountName={selectedInstanceName} jobRequestModel={LogHelper.FormatObjectForLog(jobRequestModel)}");
// load configuration for specific instance
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[selectedInstanceName];
// get client
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(selectedInstanceName, logger);
// In order to submit a new job, output asset has to be created first
var asset = await clientInstance.Assets.CreateOrUpdateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobRequestModel.OutputAssetName,
new Asset()).ConfigureAwait(false);
JobOutput[] jobOutputs = { new JobOutputAsset(jobRequestModel.OutputAssetName) };
// submit new job
var job = await clientInstance.Jobs.CreateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobRequestModel.TransformName,
jobRequestModel.JobName,
new Job
{
Input = jobRequestModel.JobInputs,
Outputs = jobOutputs,
}).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync successfully created job: job={LogHelper.FormatObjectForLog(job)}");
// create job verification request
var jobVerificationRequestModel = new JobVerificationRequestModel
{
Id = Guid.NewGuid().ToString(),
JobId = job.Id,
OriginalJobRequestModel = jobRequestModel,
MediaServiceAccountName = selectedInstanceName,
JobOutputAssetName = jobRequestModel.OutputAssetName,
JobName = job.Name,
RetryCount = 0 // initial submission, only certain number of retries are performed before skipping job verification retry,
// see job verification service for more details
};
//create job output status record
var statusInfo = MediaServicesHelper.GetJobOutputState(job, jobRequestModel.OutputAssetName);
var jobOutputStatusModel = new JobOutputStatusModel
{
Id = Guid.NewGuid().ToString(),
EventTime = statusInfo.Item2,
JobOutputState = statusInfo.Item1,
JobName = job.Name,
MediaServiceAccountName = selectedInstanceName,
JobOutputAssetName = jobRequestModel.OutputAssetName,
TransformName = jobRequestModel.TransformName
};
// in order to round robin among all healthy services, health service needs to know which instance has been used last
// data is persisted in memory only for current process
this.mediaServiceInstanceHealthService.RecordInstanceUsage(selectedInstanceName, logger);
var retryCount = 3;
var retryTimeOut = 1000;
// Job is submitted at this point, failing to do any calls after this point would result in reprocessing this job request and submitting duplicate one.
// It is OK to retry and ignore exception at the end. In current implementation based on Azure storage, it is very unlikely to fail in any of the below calls.
do
{
try
{
// persist initial job output status
await this.jobOutputStatusStorageService.CreateOrUpdateAsync(jobOutputStatusModel, logger).ConfigureAwait(false);
// persist job verification request. It is used to trigger logic to verify that job was completed and not stuck sometime in future.
var jobVerificationResult = await this.jobVerificationRequestStorageService.CreateAsync(jobVerificationRequestModel, this.verificationDelay, logger).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync successfully submitted jobVerificationModel: result={LogHelper.FormatObjectForLog(jobVerificationResult)}");
// no exception happened, let's break.
break;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
logger.LogError($"JobSchedulingService::SubmitJobAsync got exception calling jobVerificationRequestStorageService.CreateAsync: retryCount={retryCount} message={e.Message} job={LogHelper.FormatObjectForLog(job)}");
retryCount--;
await Task.Delay(retryTimeOut).ConfigureAwait(false);
}
}while (retryCount > 0);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync completed: job={LogHelper.FormatObjectForLog(job)}");
return(job);
}