private ConsolidationParams ReadConsolidationConfig()
{
var result = new ConsolidationParams();
var sql = $"select entity_type, column_name from {GetQualifiedTableName(ConsolidationConfig)}";
DatabaseUtils.EnumerateResults(ConnectionString, sql, TimeoutSecs, r =>
{
var e = EntityUtils.FromString((string)r["entity_type"]);
if (EntityUtils.CanParticipateInConsolidation(e))
{
string colName = (string)DatabaseUtils.SafeRead(r, "column_name", string.Empty);
if (!string.IsNullOrEmpty(colName))
{
var entry = new ConsolidationEntry
{
Entity = e.ToString(),
Column = colName
};
result.Entries.Add(entry);
}
}
});
result.Enabled = result.Entries.Count > 0;
return(result);
}
public static void Execute(
string connectionString,
int timeoutSecs,
int maxDegreeOfParallelism,
ConsolidationParams consolidationParams,
string stagingSchemaName)
{
if (consolidationParams.Enabled)
{
_log.Debug("Checking that consolidation keys aren't blank");
DoParallelProcessing(connectionString, timeoutSecs, maxDegreeOfParallelism, consolidationParams, stagingSchemaName);
}
}
/// <summary>
/// Updates the consolidation config table
/// </summary>
/// <param name="c">
/// The new values
/// </param>
/// <returns>
/// True if configuration was changed
/// </returns>
public bool UpdateConsolidationConfigTable(ConsolidationParams c)
{
var fromDb = ReadConsolidationConfig();
if (fromDb.DiffersFrom(c))
{
_log.Debug("Consolidation configuration has changed since last federation process");
WriteConsolidationConfig(c);
return(true);
}
return(false);
}
private static void DoParallelProcessing(
string connectionString,
int timeoutSecs,
int maxDegreeOfParallelism,
ConsolidationParams consolidationParams,
string stagingSchemaName)
{
List <Entity> entities = Enum.GetValues(typeof(Entity)).Cast <object>().Cast <Entity>().ToList();
var pOptions = new ParallelOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
};
Parallel.ForEach(entities, pOptions, (e, loopState) =>
{
if (!loopState.IsExceptional && EntityUtils.CanParticipateInConsolidation(e))
{
var entry = consolidationParams.Get(e);
if (!entry.None)
{
var qualifiedStagingTableName =
DatabaseUtils.GetQualifiedTableName(stagingSchemaName, EntityUtils.ToCtTableName(e));
var naturalKey = entry.Column;
var sql =
$"select count(1) from {qualifiedStagingTableName} where {naturalKey} is null or RTRIM(LTRIM({naturalKey})) = ''";
var count = Convert.ToInt32(DatabaseUtils.ExecuteScalar(connectionString, sql, timeoutSecs));
if (count > 0)
{
loopState.Stop();
var msg =
$"There are {count} rows in {qualifiedStagingTableName} with empty consolidation values (column='{naturalKey}')";
_log.Error(msg);
throw new ApplicationException(msg);
}
}
}
});
}
public bool ConsolidationConfigChanged(ConsolidationParams c)
{
if (DatabaseUtils.DatabaseExists(AdminConnectionString, Timeouts.AdminDatabase))
{
var fs = new FederationSchema(
AdminConnectionString,
Timeouts.AdminDatabase,
_configuration.MaxDegreeOfParallelism,
_configuration.Pipelines);
if (fs.ConsolidationConfigTableExists())
{
return(fs.ConsolidationConfigChanged(c));
}
}
return(false);
}
/// <summary>
/// Checks that the column names specified in the configuration, consolidation
/// section are valid for the given entity
/// </summary>
public static void CheckNaturalKeyColumnsInConfiguration(ConsolidationParams consolidationParams)
{
_log.Debug("Checking that consolidation key columns are valid in configuration");
StagingTablesBuilder b = StagingTablesBuilder.Get(StagingSchema.PrimaryStagingSchemaName);
if (consolidationParams.Enabled)
{
var entitiesUsed = new List <Entity>();
foreach (Entity entity in Enum.GetValues(typeof(Entity)))
{
if (EntityUtils.CanParticipateInConsolidation(entity))
{
if (entitiesUsed.Contains(entity))
{
throw new ApplicationException(string.Format("Entity declared more than once in consolidation configuration: {0}", entity));
}
entitiesUsed.Add(entity);
var entry = consolidationParams.Get(entity);
if (entry != null && !entry.None)
{
string stagingTableName = EntityUtils.ToCtTableName(entity);
var table = b.GetTable(stagingTableName);
if (!table.ColumnExists(entry.Column))
{
throw new ApplicationException(string.Format("The specified consolidation column ({0}) does not exist in the entity: {1}", entry.Column, entity));
}
}
}
}
}
}
private void WriteConsolidationConfig(ConsolidationParams c)
{
var qualifiedConfigTable = GetQualifiedTableName(ConsolidationConfig);
var sql = $"delete from {qualifiedConfigTable}";
DatabaseUtils.ExecuteSql(ConnectionString, sql, TimeoutSecs);
if (c.Enabled)
{
foreach (var entry in c.Entries)
{
var e = EntityUtils.FromString(entry.Entity);
if (EntityUtils.CanParticipateInConsolidation(e))
{
sql = $"insert into {qualifiedConfigTable} (entity_type, column_name) values (@E, @C)";
SqlParameter[] p = { new SqlParameter("@E", entry.Entity), new SqlParameter("@C", entry.Column) };
DatabaseUtils.ExecuteSql(ConnectionString, sql, TimeoutSecs, p);
}
}
}
}
public bool ConsolidationConfigChanged(ConsolidationParams c)
{
var fromDb = ReadConsolidationConfig();
return(fromDb.DiffersFrom(c));
}
private void CheckForChangeInConsolidationConfig(AdminDatabase adminDb, PublicDatabase publicDb, ConsolidationParams consolidation)
{
if (adminDb.ConsolidationConfigChanged(consolidation) && publicDb.TablesExist())
{
throw new ApplicationException("Consolidation configuration has changed. Please rebuild the data store by deleting ADMIN and PUBLIC databases");
}
}
