/// <summary>
/// Stores job output status record to storage service
/// </summary>
/// <param name="mediaServiceAccountName">Account name to load</param>
/// <param name="jobOutputStatusModel">Job output status to update</param>
/// <param name="job">Job data loaded from API</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Task for async operation</returns>
private async Task UpdateJobOutputStatusAsync(string mediaServiceAccountName, JobOutputStatusModel jobOutputStatusModel, Job job, ILogger logger)
{
if (job != null)
{
var statusInfo = MediaServicesHelper.GetJobOutputState(job, jobOutputStatusModel.JobOutputAssetName);
var jobOutputStatusModelFromAPI = new JobOutputStatusModel
{
Id = Guid.NewGuid().ToString(),
EventTime = statusInfo.Item2,
JobOutputState = statusInfo.Item1,
JobName = jobOutputStatusModel.JobName,
MediaServiceAccountName = mediaServiceAccountName,
JobOutputAssetName = jobOutputStatusModel.JobOutputAssetName,
TransformName = jobOutputStatusModel.TransformName
};
// Provisioning request is created for all job output status events that are finished.
if (jobOutputStatusModelFromAPI.JobOutputState == JobState.Finished)
{
var provisioningRequestResult = await this.provisioningRequestStorageService.CreateAsync(
new ProvisioningRequestModel
{
Id = Guid.NewGuid().ToString(),
ProcessedAssetMediaServiceAccountName = jobOutputStatusModelFromAPI.MediaServiceAccountName,
ProcessedAssetName = jobOutputStatusModelFromAPI.JobOutputAssetName,
StreamingLocatorName = $"streaming-{jobOutputStatusModelFromAPI.JobOutputAssetName}"
},
logger).ConfigureAwait(false);
logger.LogInformation($"JobOutputStatusSyncService::UpdateJobOutputStatusAsync created stream provisioning request: result={LogHelper.FormatObjectForLog(provisioningRequestResult)}");
}
await this.jobOutputStatusStorageService.CreateOrUpdateAsync(jobOutputStatusModelFromAPI, logger).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies the status of given job, implements business logic to resubmit jobs if needed
/// </summary>
/// <param name="jobVerificationRequestModel">Job verification request</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Processed job verification request</returns>
public async Task <JobVerificationRequestModel> VerifyJobAsync(JobVerificationRequestModel jobVerificationRequestModel, ILogger logger)
{
logger.LogInformation($"JobVerificationService::VerifyJobAsync started: jobVerificationRequestModel={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
// Get latest job output status from storage service.
var jobOutputStatus = await this.jobOutputStatusStorageService.GetLatestJobOutputStatusAsync(jobVerificationRequestModel.JobName, jobVerificationRequestModel.JobOutputAssetName).ConfigureAwait(false);
var jobOutputStatusLoadedFromAPI = false;
// if job has not reached final state, need to reload status from Azure Media Service APIs in case of delayed or lost EventGrid event.
if (jobOutputStatus?.JobOutputState != JobState.Finished && jobOutputStatus?.JobOutputState != JobState.Error && jobOutputStatus?.JobOutputState != JobState.Canceled)
{
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[jobVerificationRequestModel.MediaServiceAccountName];
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(jobVerificationRequestModel.MediaServiceAccountName, logger);
logger.LogInformation($"JobVerificationService::VerifyJobAsync checking job status using API: mediaServiceInstanceName={jobVerificationRequestModel.MediaServiceAccountName}");
// Get job data to verify status of specific job output.
var job = await clientInstance.Jobs.GetAsync(clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobVerificationRequestModel.OriginalJobRequestModel.TransformName,
jobVerificationRequestModel.JobName).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync loaded job data from API: job={LogHelper.FormatObjectForLog(job)}");
if (job != null)
{
// create job output status record using job loaded from Azure Media Service API.
var statusInfo = MediaServicesHelper.GetJobOutputState(job, jobVerificationRequestModel.JobOutputAssetName);
jobOutputStatus = new JobOutputStatusModel
{
Id = Guid.NewGuid().ToString(),
EventTime = statusInfo.Item2,
JobOutputState = statusInfo.Item1,
JobName = job.Name,
MediaServiceAccountName = jobVerificationRequestModel.MediaServiceAccountName,
JobOutputAssetName = jobVerificationRequestModel.JobOutputAssetName,
TransformName = jobVerificationRequestModel.OriginalJobRequestModel.TransformName,
HasRetriableError = MediaServicesHelper.HasRetriableError(job, jobVerificationRequestModel.JobOutputAssetName) // check if job should be retried
};
jobOutputStatusLoadedFromAPI = true;
// persist job output status record
await this.jobOutputStatusStorageService.CreateOrUpdateAsync(jobOutputStatus, logger).ConfigureAwait(false);
}
}
// At this point here, jobOutputStatus is either loaded from job output status storage or from Azure Media Service API.
logger.LogInformation($"JobVerificationService::VerifyJobAsync jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
// Check if job output has been successfully finished.
if (jobOutputStatus?.JobOutputState == JobState.Finished)
{
await this.ProcessFinishedJobAsync(jobVerificationRequestModel, jobOutputStatusLoadedFromAPI, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job was completed successfully: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// Check if job output failed.
if (jobOutputStatus?.JobOutputState == JobState.Error)
{
await this.ProcessFailedJob(jobVerificationRequestModel, jobOutputStatus, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job failed: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// check if job has been canceled.
if (jobOutputStatus?.JobOutputState == JobState.Canceled)
{
logger.LogInformation($"JobVerificationService::VerifyJobAsync] job canceled: jobOutputStatus={LogHelper.FormatObjectForLog(jobOutputStatus)}");
return(jobVerificationRequestModel);
}
// At this point, job is stuck, it is not in the final state and long enough time period has passed (since this code is running for a given job).
await this.ProcessStuckJob(jobVerificationRequestModel, logger).ConfigureAwait(false);
logger.LogInformation($"JobVerificationService::VerifyJobAsync completed: job={LogHelper.FormatObjectForLog(jobVerificationRequestModel)}");
return(jobVerificationRequestModel);
}
/// <summary>
/// Submits job to Azure Media Services.
/// </summary>
/// <param name="jobRequestModel">Job to submit.</param>
/// <param name="logger">Logger to log data</param>
/// <returns>Submitted job</returns>
public async Task <Job> SubmitJobAsync(JobRequestModel jobRequestModel, ILogger logger)
{
logger.LogInformation($"JobSchedulingService::SubmitJobAsync started: jobRequestModel={LogHelper.FormatObjectForLog(jobRequestModel)}");
// Get next available Azure Media Services instance
var selectedInstanceName = await this.mediaServiceInstanceHealthService.GetNextAvailableInstanceAsync(logger).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync selected healthy instance: MediaServiceAccountName={selectedInstanceName} jobRequestModel={LogHelper.FormatObjectForLog(jobRequestModel)}");
// load configuration for specific instance
var clientConfiguration = this.configService.MediaServiceInstanceConfiguration[selectedInstanceName];
// get client
var clientInstance = this.mediaServiceInstanceFactory.GetMediaServiceInstance(selectedInstanceName, logger);
// In order to submit a new job, output asset has to be created first
var asset = await clientInstance.Assets.CreateOrUpdateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobRequestModel.OutputAssetName,
new Asset()).ConfigureAwait(false);
JobOutput[] jobOutputs = { new JobOutputAsset(jobRequestModel.OutputAssetName) };
// submit new job
var job = await clientInstance.Jobs.CreateAsync(
clientConfiguration.ResourceGroup,
clientConfiguration.AccountName,
jobRequestModel.TransformName,
jobRequestModel.JobName,
new Job
{
Input = jobRequestModel.JobInputs,
Outputs = jobOutputs,
}).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync successfully created job: job={LogHelper.FormatObjectForLog(job)}");
// create job verification request
var jobVerificationRequestModel = new JobVerificationRequestModel
{
Id = Guid.NewGuid().ToString(),
JobId = job.Id,
OriginalJobRequestModel = jobRequestModel,
MediaServiceAccountName = selectedInstanceName,
JobOutputAssetName = jobRequestModel.OutputAssetName,
JobName = job.Name,
RetryCount = 0 // initial submission, only certain number of retries are performed before skipping job verification retry,
// see job verification service for more details
};
//create job output status record
var statusInfo = MediaServicesHelper.GetJobOutputState(job, jobRequestModel.OutputAssetName);
var jobOutputStatusModel = new JobOutputStatusModel
{
Id = Guid.NewGuid().ToString(),
EventTime = statusInfo.Item2,
JobOutputState = statusInfo.Item1,
JobName = job.Name,
MediaServiceAccountName = selectedInstanceName,
JobOutputAssetName = jobRequestModel.OutputAssetName,
TransformName = jobRequestModel.TransformName
};
// in order to round robin among all healthy services, health service needs to know which instance has been used last
// data is persisted in memory only for current process
this.mediaServiceInstanceHealthService.RecordInstanceUsage(selectedInstanceName, logger);
var retryCount = 3;
var retryTimeOut = 1000;
// Job is submitted at this point, failing to do any calls after this point would result in reprocessing this job request and submitting duplicate one.
// It is OK to retry and ignore exception at the end. In current implementation based on Azure storage, it is very unlikely to fail in any of the below calls.
do
{
try
{
// persist initial job output status
await this.jobOutputStatusStorageService.CreateOrUpdateAsync(jobOutputStatusModel, logger).ConfigureAwait(false);
// persist job verification request. It is used to trigger logic to verify that job was completed and not stuck sometime in future.
var jobVerificationResult = await this.jobVerificationRequestStorageService.CreateAsync(jobVerificationRequestModel, this.verificationDelay, logger).ConfigureAwait(false);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync successfully submitted jobVerificationModel: result={LogHelper.FormatObjectForLog(jobVerificationResult)}");
// no exception happened, let's break.
break;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
logger.LogError($"JobSchedulingService::SubmitJobAsync got exception calling jobVerificationRequestStorageService.CreateAsync: retryCount={retryCount} message={e.Message} job={LogHelper.FormatObjectForLog(job)}");
retryCount--;
await Task.Delay(retryTimeOut).ConfigureAwait(false);
}
}while (retryCount > 0);
logger.LogInformation($"JobSchedulingService::SubmitJobAsync completed: job={LogHelper.FormatObjectForLog(job)}");
return(job);
}