private async Task UpdateStage(PublishReleaseDataMessage message, ReleasePublishingStatusDataStage stage,
ReleasePublishingStatusLogMessage?logMessage = null)
{
await _releasePublishingStatusService.UpdateDataStageAsync(message.ReleaseId, message.ReleaseStatusId,
stage,
logMessage);
}
private static IDictionary <string, object> BuildPipelineParameters(PublishReleaseDataMessage message)
{
return(new Dictionary <string, object>
{
{ "releaseId", message.ReleaseId },
{ "releaseStatusId", message.ReleaseStatusId }
});
}
private static bool TriggerDataFactoryReleasePipeline(DataFactoryClientConfiguration configuration,
ILogger logger, PublishReleaseDataMessage message)
{
var parameters = BuildPipelineParameters(message);
var client = GetDataFactoryClient(configuration);
var runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(configuration.ResourceGroupName,
configuration.DataFactoryName, configuration.PipelineName, parameters: parameters).Result;
logger.LogInformation(
$"Pipeline status code: {runResponse.Response.StatusCode}, run Id: {runResponse.Body.RunId}");
return(runResponse.Response.IsSuccessStatusCode);
}
public async Task PublishReleaseData(
[QueueTrigger(PublishReleaseDataQueue)]
PublishReleaseDataMessage message,
ExecutionContext executionContext,
ILogger logger)
{
logger.LogInformation("{0} triggered: {1}",
executionContext.FunctionName,
message);
try
{
// Azure Data Factory isn't emulated for running in a local environment
// It also has an overhead to run which isn't necessary if there are no data files
var runDataFactory = !EnvironmentUtils.IsLocalEnvironment() &&
await ReleaseHasAnyDataFiles(message.ReleaseId);
if (runDataFactory)
{
var clientConfiguration = new DataFactoryClientConfiguration(_configuration);
var success = TriggerDataFactoryReleasePipeline(clientConfiguration, logger, message);
await UpdateStage(message, success?Started : Failed);
}
else
{
await SimulateDataFactoryReleasePipeline(message);
await UpdateStage(message, Complete);
}
}
catch (Exception e)
{
logger.LogError(e, "Exception occured while executing {0}",
executionContext.FunctionName);
await UpdateStage(message, Failed,
new ReleasePublishingStatusLogMessage($"Exception in data stage: {e.Message}"));
}
logger.LogInformation("{0} completed",
executionContext.FunctionName);
}
// ReSharper disable once UnusedMember.Global
public async Task PublishReleaseData(
[QueueTrigger(PublishReleaseDataQueue)] PublishReleaseDataMessage message,
ExecutionContext executionContext,
ILogger logger)
{
logger.LogInformation($"{executionContext.FunctionName} triggered: {message}");
if (PublisherUtils.IsDevelopment())
{
// Skip the ADF Pipeline if running locally
// If the Release is immediate then trigger publishing the content
// This usually happens when the ADF Pipeline is complete
if (await _releaseStatusService.IsImmediate(message.ReleaseId, message.ReleaseStatusId))
{
await _queueService.QueuePublishReleaseContentMessageAsync(message.ReleaseId,
message.ReleaseStatusId);
}
await UpdateStage(message, Complete);
}
else
{
try
{
var clientConfiguration = new DataFactoryClientConfiguration(_configuration);
var success = TriggerDataFactoryReleasePipeline(clientConfiguration, logger, message);
await UpdateStage(message, success?Started : Failed);
}
catch (Exception e)
{
logger.LogError(e, $"Exception occured while executing {executionContext.FunctionName}");
await UpdateStage(message, Failed,
new ReleaseStatusLogMessage($"Exception in data stage: {e.Message}"));
}
}
logger.LogInformation($"{executionContext.FunctionName} completed");
}
private async Task SimulateDataFactoryReleasePipeline(PublishReleaseDataMessage message)
{
// Create the Public Statistics Release if a Statistics Release exists.
// This copying would have happened in the ADF Pipeline in procedure DropAndCreateRelease.
// The Statistics Release will exist if Subjects have been imported previously
// (to either to this Release or a previous version), despite there being no Subjects now.
// TODO EES-2819 We should call the DropAndCreateRelease procedure here or replicate its behaviour instead
// of just creating the Release. Reason:
// If this stage is ever forcefully retried then it's possible the Release already exists, that it's
// attributes may now be different, and that it may have had Subjects and Footnotes that should be deleted.
// The DropAndCreateRelease procedure covers these scenarios by deleting the Release and its related data.
await _releaseService.CreatePublicStatisticsRelease(message.ReleaseId);
// If the Release is immediate then trigger publishing the content.
// This would have happened when the ADF Pipeline completed in the callback it makes to
// DataFactoryPipelineStatusFunction.
if (await _releasePublishingStatusService.IsImmediate(message.ReleaseId, message.ReleaseStatusId))
{
await _queueService.QueuePublishReleaseContentMessageAsync(message.ReleaseId,
message.ReleaseStatusId);
}
}
