private void DoParallelMarkHistoryApplied(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
MarkHistoryRowsAsApplied(table.Name);
}
});
}
private void DoParallelProcessingFillWeeks(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
var cols = b.GetColumnNames(table.Name);
if (cols.Contains("weeks") && cols.Contains("federated_event_id") && !table.Name.Equals("CT_EVENT"))
{
FillWeeksCol(table);
}
}
});
}
private void GenerateMappings()
{
var b = PublicStagingTablesBuilder.Get();
foreach (var table in b.GetTables())
{
Debug.Assert(table.Name.StartsWith("CT_"));
var publicTableName = GetPublicTableName(table.Name);
var publicTable = Find(publicTableName);
_mappings.Add(new TableMapping {
PublicStagingTable = table, PublicTable = publicTable
});
}
}
private void CheckAllChangesApplied()
{
// throw if any rows in the staging tables are as yet untransformed (i.e. they're still there)...
var b = PublicStagingTablesBuilder.Get();
var pOptions = new ParallelOptions {
MaxDegreeOfParallelism = _configuration.MaxDegreeOfParallelism
};
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
var sql = $"select count(1) from {table.QualifiedName}";
DatabaseUtils.GetSingleResult(PublicConnectionString, sql, Timeouts.PublicDatabase, r =>
{
var recCount = (int)r[0];
if (recCount > 0)
{
throw new ApplicationException($"{recCount} staging rows not yet applied: {table.Name}");
}
});
});
}
private RowCountAndDuration DoParallelProcessingCreateStage(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
RowCountAndDuration result = new RowCountAndDuration();
object locker = new object();
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
var p = new PublicStagingEtlProcess(
table, AdminConnectionString, PublicConnectionString, Timeouts.PublicDatabase, _configuration.Pipelines);
p.Execute();
var errors = p.GetAllErrors().ToArray();
if (errors.Any())
{
loopState.Stop();
string msg = $"Errors occurred during execution of public staging process: {table.Name}";
_log.Error(msg);
// throw the first exception
throw new ApplicationException(msg, errors[0]);
}
lock (locker)
{
result += p.Stats;
}
}
});
return(result);
}
public void PopulateStage(Guid adminAppKey)
{
OnProgressEvent(new VertoProgressEventArgs {
ProgressString = "Populating public stage", Section = ProcessingSection.StagingPublic
});
_log.Debug("Populating public stage");
CheckPreconditions(adminAppKey);
var b = PublicStagingTablesBuilder.Get();
var pOptions = new ParallelOptions {
MaxDegreeOfParallelism = _configuration.MaxDegreeOfParallelism
};
var stats = DoParallelProcessingCreateStage(b, pOptions);
DoParallelProcessingFillWeeks(b, pOptions);
if (stats.RowCount > 0)
{
// only mark as applied _after_ the PUBLIC staging schema has been successfully populated...
DoParallelMarkHistoryApplied(b, pOptions);
}
}