private void DoParallelMarkHistoryApplied(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
MarkHistoryRowsAsApplied(table.Name);
}
});
}
private void DoParallelProcessingFillWeeks(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
var cols = b.GetColumnNames(table.Name);
if (cols.Contains("weeks") && cols.Contains("federated_event_id") && !table.Name.Equals("CT_EVENT"))
{
FillWeeksCol(table);
}
}
});
}
private RowCountAndDuration DoParallelProcessingCreateStage(PublicStagingTablesBuilder b, ParallelOptions pOptions)
{
RowCountAndDuration result = new RowCountAndDuration();
object locker = new object();
Parallel.ForEach(b.GetTables(), pOptions, (table, loopState) =>
{
if (!loopState.IsExceptional)
{
var p = new PublicStagingEtlProcess(
table, AdminConnectionString, PublicConnectionString, Timeouts.PublicDatabase, _configuration.Pipelines);
p.Execute();
var errors = p.GetAllErrors().ToArray();
if (errors.Any())
{
loopState.Stop();
string msg = $"Errors occurred during execution of public staging process: {table.Name}";
_log.Error(msg);
// throw the first exception
throw new ApplicationException(msg, errors[0]);
}
lock (locker)
{
result += p.Stats;
}
}
});
return(result);
}