private List <string> ApplyChangesToDatabase_Insert(List <ConceptApplication> toBeInserted, List <ConceptApplication> newApplications)
{
var newScripts = new List <string>();
Graph.TopologicalSort(toBeInserted, ConceptApplication.GetDependencyPairs(newApplications));
int insertedCACount = 0;
foreach (var ca in toBeInserted)
{
string[] createSqlScripts = SplitSqlScript(ca.CreateQuery);
if (createSqlScripts.Length > 0)
{
LogDatabaseChanges(ca, "Creating");
insertedCACount++;
}
newScripts.AddRange(createSqlScripts);
newScripts.AddRange(_sqlTransactionBatches.JoinScripts(_conceptApplicationRepository.InsertMetadataSql(ca)));
newScripts.AddRange(MaybeCommitMetadataAfterDdl(createSqlScripts));
}
_logger.Info($"Creating {insertedCACount} concept applications.");
return(newScripts);
}
/// <summary>
/// Compares current generated database model (from DSL scripts)
/// and the previously created objects in database (from metadata in table Rhetos.AppliedConcept).
/// </summary>
public DatabaseDiff Diff()
{
var stopwatch = Stopwatch.StartNew();
var oldApplications = _conceptApplicationRepository.Load();
_performanceLogger.Write(stopwatch, "Loaded old concept applications.");
var newApplications = ConceptApplication.FromDatabaseObjects(_databaseModel.DatabaseObjects);
_performanceLogger.Write(stopwatch, "Got new concept applications.");
MatchAndComputeNewApplicationIds(oldApplications, newApplications);
_performanceLogger.Write(stopwatch, "Match new and old concept applications.");
ConceptApplication.CheckKeyUniqueness(newApplications, "generated, after matching");
_performanceLogger.Write(stopwatch, "Verify new concept applications' integrity.");
newApplications = TrimEmptyApplications(newApplications);
_performanceLogger.Write(stopwatch, "Removed unused concept applications.");
var diff = CalculateApplicationsToBeRemovedAndInserted(oldApplications, newApplications);
_performanceLogger.Write(stopwatch, "Analyzed differences in database structure.");
return(diff);
}
public List <string> DeleteMetadataSql(ConceptApplication oldCA)
{
return(new List <string>
{
Sql.Format("ConceptApplicationRepository_Delete", SqlUtility.QuoteGuid(oldCA.Id))
});
}
private List <SqlBatchScript> ApplyChangesToDatabase_Insert(List <ConceptApplication> toBeInserted, List <ConceptApplication> newApplications)
{
var newScripts = new List <SqlBatchScript>();
Graph.TopologicalSort(toBeInserted, ConceptApplication.GetDependencyPairs(newApplications));
int insertedCACount = 0;
foreach (var ca in toBeInserted)
{
if (!string.IsNullOrWhiteSpace(ca.CreateQuery))
{
LogDatabaseChanges(ca, "Creating");
insertedCACount++;
}
newScripts.Add(new SqlBatchScript {
Sql = ca.CreateQuery, IsBatch = true, Name = $"Creating {ca}."
});
newScripts.AddRange(_sqlTransactionBatches.JoinScripts(_conceptApplicationRepository.InsertMetadataSql(ca))
.Select((sql, x) => new SqlBatchScript {
Sql = sql, IsBatch = false, Name = $"Creating {ca} metadata ({x + 1})."
}));
newScripts.AddRange(MaybeCommitMetadataAfterDdl(ca.CreateQuery));
}
_logger.Info($"Creating {insertedCACount} database objects.");
return(newScripts);
}
private List <SqlBatchScript> ApplyChangesToDatabase_Remove(List <ConceptApplication> toBeRemoved, List <ConceptApplication> oldApplications)
{
var newScripts = new List <SqlBatchScript>();
toBeRemoved = toBeRemoved.OrderBy(ca => ca.OldCreationOrder).ToList(); // TopologicalSort is stable sort, so it will keep this (original) order unless current dependencies direct otherwise.
Graph.TopologicalSort(toBeRemoved, ConceptApplication.GetDependencyPairs(oldApplications)); // Concept's dependencies might have changed, without dropping and recreating the concept. It is important to compute up-to-date remove order, otherwise FK constraint FK_AppliedConceptDependsOn_DependsOn might fail.
toBeRemoved.Reverse();
int removedCACount = 0;
foreach (var ca in toBeRemoved)
{
if (!string.IsNullOrWhiteSpace(ca.RemoveQuery))
{
LogDatabaseChanges(ca, "Removing");
removedCACount++;
}
newScripts.Add(new SqlBatchScript {
Sql = ca.RemoveQuery, IsBatch = true, Name = $"Removing {ca}."
});
newScripts.AddRange(_sqlTransactionBatches.JoinScripts(_conceptApplicationRepository.DeleteMetadataSql(ca))
.Select((sql, x) => new SqlBatchScript {
Sql = sql, IsBatch = false, Name = $"Removing {ca} metadata ({x + 1})."
}));
newScripts.AddRange(MaybeCommitMetadataAfterDdl(ca.RemoveQuery));
}
_logger.Info($"Removing {removedCACount} database objects.");
return(newScripts);
}
private List <string> ApplyChangesToDatabase_Remove(List <ConceptApplication> toBeRemoved, List <ConceptApplication> oldApplications)
{
var newScripts = new List <string>();
toBeRemoved = toBeRemoved.OrderBy(ca => ca.OldCreationOrder).ToList(); // TopologicalSort is stable sort, so it will keep this (original) order unless current dependencies direct otherwise.
Graph.TopologicalSort(toBeRemoved, ConceptApplication.GetDependencyPairs(oldApplications)); // Concept's dependencies might have changed, without dropping and recreating the concept. It is important to compute up-to-date remove order, otherwise FK constraint FK_AppliedConceptDependsOn_DependsOn might fail.
toBeRemoved.Reverse();
int removedCACount = 0;
foreach (var ca in toBeRemoved)
{
string[] removeSqlScripts = SplitSqlScript(ca.RemoveQuery);
if (removeSqlScripts.Length > 0)
{
LogDatabaseChanges(ca, "Removing");
removedCACount++;
}
newScripts.AddRange(removeSqlScripts);
newScripts.AddRange(_sqlTransactionBatches.JoinScripts(_conceptApplicationRepository.DeleteMetadataSql(ca)));
newScripts.AddRange(MaybeCommitMetadataAfterDdl(removeSqlScripts));
}
_logger.Info($"Removing {removedCACount} concept applications.");
return(newScripts);
}
public List<string> DeleteMetadataSql(ConceptApplication ca)
{
return new List<string>
{
Sql.Format("ConceptApplicationRepository_Delete", SqlUtility.QuoteGuid(ca.Id))
};
}
private void LogDatabaseChanges(ConceptApplication conceptApplication, string action, Func <string> additionalInfo = null)
{
_changesLogger.Trace("{0} {1}, ID={2}.{3}{4}",
action,
conceptApplication.GetConceptApplicationKey(),
SqlUtility.GuidToString(conceptApplication.Id),
additionalInfo != null ? " " : null,
additionalInfo != null ? additionalInfo() : null);
}
public List <ConceptApplication> Load()
{
var previoslyAppliedConcepts = LoadOldConceptApplicationsFromDatabase();
ConceptApplication.CheckKeyUniqueness(previoslyAppliedConcepts, "loaded");
var dependencies = LoadDependenciesFromDatabase();
EvaluateDependencies(previoslyAppliedConcepts, dependencies); // Replace GUIDs with actual ConceptApplication instances.
return(previoslyAppliedConcepts);
}
private List <ConceptApplication> TrimEmptyApplications(List <ConceptApplication> newApplications)
{
var emptyCreateQuery = newApplications.Where(ca => string.IsNullOrWhiteSpace(ca.CreateQuery)).ToList();
var emptyCreateHasRemove = emptyCreateQuery.FirstOrDefault(ca => !string.IsNullOrWhiteSpace(ca.RemoveQuery));
if (emptyCreateHasRemove != null)
{
throw new FrameworkException("A concept that does not create database objects (CreateDatabaseStructure) cannot remove them (RemoveDatabaseStructure): "
+ emptyCreateHasRemove.GetConceptApplicationKey() + ".");
}
var removeLeaves = Graph.RemovableLeaves(emptyCreateQuery, ConceptApplication.GetDependencyPairs(newApplications));
_logger.Trace(() => $"Removing {removeLeaves.Count} empty leaf concept applications:{string.Concat(removeLeaves.Select(l => "\r\n-" + l))}");
return(newApplications.Except(removeLeaves).ToList());
}
public List <string> UpdateMetadataSql(NewConceptApplication ca, ConceptApplication oldApp)
{
var sql = new List <string>();
if (oldApp.RemoveQuery != ca.RemoveQuery)
{
sql.Add(Sql.Format("ConceptApplicationRepository_Update",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteText(ca.ConceptInfoTypeName),
SqlUtility.QuoteText(ca.ConceptInfoKey),
SqlUtility.QuoteText(ca.ConceptImplementationTypeName),
SqlUtility.QuoteText(_xmlUtility.SerializeToXml(ca.ConceptInfo)),
SqlUtility.QuoteText(ca.CreateQuery),
SqlUtility.QuoteText(ca.RemoveQuery),
SqlUtility.QuoteText(ca.ConceptImplementationVersion.ToString())));
}
HashSet <Guid> oldDependsOn = new HashSet <Guid>(oldApp.DependsOn.Select(depOn => depOn.ConceptApplication.Id));
HashSet <Guid> newDependsOn = new HashSet <Guid>(ca.DependsOn.Select(depOn => depOn.ConceptApplication.Id));
foreach (var dependsOnId in ca.DependsOn.Select(d => d.ConceptApplication.Id).Distinct())
{
if (!oldDependsOn.Contains(dependsOnId))
{
sql.Add(Sql.Format("ConceptApplicationRepository_InsertDependency",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteGuid(dependsOnId)));
}
}
foreach (var dependsOnId in oldApp.DependsOn.Select(d => d.ConceptApplication.Id).Distinct())
{
if (!newDependsOn.Contains(dependsOnId))
{
sql.Add(Sql.Format("ConceptApplicationRepository_DeleteDependency",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteGuid(dependsOnId)));
}
}
return(sql);
}
public List <string> InsertMetadataSql(ConceptApplication newCA)
{
var sql = new List <string>();
sql.Add(Sql.Format("ConceptApplicationRepository_Insert",
SqlUtility.QuoteGuid(newCA.Id),
SqlUtility.QuoteText(newCA.ConceptInfoTypeName),
SqlUtility.QuoteText(newCA.ConceptInfoKey),
SqlUtility.QuoteText(newCA.ConceptImplementationTypeName),
SqlUtility.QuoteText(newCA.CreateQuery),
SqlUtility.QuoteText(newCA.RemoveQuery)));
foreach (var dependsOnId in newCA.DependsOnConceptApplications.Select(d => d.Id).Distinct())
{
sql.Add(Sql.Format("ConceptApplicationRepository_InsertDependency",
SqlUtility.QuoteGuid(newCA.Id),
SqlUtility.QuoteGuid(dependsOnId)));
}
return(sql);
}
protected void Log(ConceptApplication conceptApplication, string action)
{
_logger.Info("{0} {1}, ID={2}.", action, conceptApplication.GetConceptApplicationKey(), SqlUtility.GuidToString(conceptApplication.Id));
}
public List<string> UpdateMetadataSql(NewConceptApplication ca, ConceptApplication oldApp)
{
var sql = new List<string>();
if (oldApp.RemoveQuery != ca.RemoveQuery)
sql.Add(Sql.Format("ConceptApplicationRepository_Update",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteText(ca.ConceptInfoTypeName),
SqlUtility.QuoteText(ca.ConceptInfoKey),
SqlUtility.QuoteText(ca.ConceptImplementationTypeName),
SqlUtility.QuoteText(_xmlUtility.SerializeToXml(ca.ConceptInfo)),
SqlUtility.QuoteText(ca.CreateQuery),
SqlUtility.QuoteText(ca.RemoveQuery),
SqlUtility.QuoteText(ca.ConceptImplementationVersion.ToString())));
HashSet<Guid> oldDependsOn = new HashSet<Guid>(oldApp.DependsOn.Select(depOn => depOn.ConceptApplication.Id));
HashSet<Guid> newDependsOn = new HashSet<Guid>(ca.DependsOn.Select(depOn => depOn.ConceptApplication.Id));
foreach (var dependsOnId in ca.DependsOn.Select(d => d.ConceptApplication.Id).Distinct())
if (!oldDependsOn.Contains(dependsOnId))
sql.Add(Sql.Format("ConceptApplicationRepository_InsertDependency",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteGuid(dependsOnId)));
foreach (var dependsOnId in oldApp.DependsOn.Select(d => d.ConceptApplication.Id).Distinct())
if (!newDependsOn.Contains(dependsOnId))
sql.Add(Sql.Format("ConceptApplicationRepository_DeleteDependency",
SqlUtility.QuoteGuid(ca.Id),
SqlUtility.QuoteGuid(dependsOnId)));
return sql;
}
public static string DeleteMetadataSql(ConceptApplication ca)
{
return Sql.Format("ConceptApplicationRepository_Delete", SqlUtility.QuoteGuid(ca.Id));
}
protected static void AddConceptApplicationSeparator(ConceptApplication ca, CodeBuilder sqlCodeBuilder)
{
sqlCodeBuilder.InsertCode(string.Format("{0}{1}{2}{3}",
NextConceptApplicationSeparator, NextConceptApplicationIdPrefix, ca.Id, NextConceptApplicationIdSuffix));
}
protected void LogDatabaseChanges(ConceptApplication conceptApplication, string action, Func<string> additionalInfo = null)
{
_conceptsLogger.Trace("{0} {1}, ID={2}.{3}{4}",
action,
conceptApplication.GetConceptApplicationKey(),
SqlUtility.GuidToString(conceptApplication.Id),
additionalInfo != null ? " " : null,
additionalInfo != null ? additionalInfo() : null);
}
private DatabaseDiff CalculateApplicationsToBeRemovedAndInserted(List <ConceptApplication> oldApplications, List <ConceptApplication> newApplications)
{
var oldApplicationsByKey = oldApplications.ToDictionary(a => a.GetConceptApplicationKey());
var newApplicationsByKey = newApplications.ToDictionary(a => a.GetConceptApplicationKey());
// Find directly inserted and removed concept applications:
var directlyRemoved = oldApplicationsByKey.Keys.Except(newApplicationsByKey.Keys).ToList();
var directlyInserted = newApplicationsByKey.Keys.Except(oldApplicationsByKey.Keys).ToList();
foreach (string ca in directlyRemoved)
{
_logger.Trace("Directly removed concept application: " + ca);
}
foreach (string ca in directlyInserted)
{
_logger.Trace("Directly inserted concept application: " + ca);
}
// Find changed concept applications (different create SQL query):
var existingApplications = oldApplicationsByKey.Keys.Intersect(newApplicationsByKey.Keys).ToList();
var changedApplications = existingApplications
.Where(appKey => !string.Equals(
oldApplicationsByKey[appKey].CreateQuery,
newApplicationsByKey[appKey].CreateQuery,
StringComparison.Ordinal))
.ToList();
// Find dependent concepts applications to be regenerated:
var toBeRemovedKeys = directlyRemoved.Union(changedApplications).ToList();
var oldDependencies = ConceptApplication.GetDependencyPairs(oldApplications).Select(dep => Tuple.Create(dep.Item1.GetConceptApplicationKey(), dep.Item2.GetConceptApplicationKey()));
var dependentRemovedApplications = Graph.IncludeDependents(toBeRemovedKeys, oldDependencies).Except(toBeRemovedKeys);
var toBeInsertedKeys = directlyInserted.Union(changedApplications).ToList();
var newDependencies = ConceptApplication.GetDependencyPairs(newApplications).Select(dep => Tuple.Create(dep.Item1.GetConceptApplicationKey(), dep.Item2.GetConceptApplicationKey()));
var dependentInsertedApplications = Graph.IncludeDependents(toBeInsertedKeys, newDependencies).Except(toBeInsertedKeys);
var refreshDependents = dependentRemovedApplications.Union(dependentInsertedApplications).ToList();
toBeRemovedKeys.AddRange(refreshDependents.Intersect(oldApplicationsByKey.Keys));
toBeInsertedKeys.AddRange(refreshDependents.Intersect(newApplicationsByKey.Keys));
// Report dependencies for items that need to be refreshed, for logging and debugging only:
var newDependenciesByDependent = newDependencies.GroupBy(dep => dep.Item2, dep => dep.Item1).ToDictionary(group => group.Key, group => group.ToList());
var oldDependenciesByDependent = oldDependencies.GroupBy(dep => dep.Item2, dep => dep.Item1).ToDictionary(group => group.Key, group => group.ToList());
var toBeInsertedIndex = new HashSet <string>(toBeInsertedKeys);
var toBeRemovedIndex = new HashSet <string>(toBeRemovedKeys);
var changedApplicationsIndex = new HashSet <string>(changedApplications);
var refreshes = new List <(ConceptApplication RefreshedConcept, ConceptApplication Dependency, RefreshDependencyStatus DependencyStatus)>();
foreach (string ca in refreshDependents.Intersect(newApplicationsByKey.Keys))
{
var refreshedConcept = newApplicationsByKey[ca];
var refreshBecauseNew = new HashSet <string>(newDependenciesByDependent.GetValueOrEmpty(ca).Intersect(toBeInsertedIndex));
var refreshBecauseOld = new HashSet <string>(oldDependenciesByDependent.GetValueOrEmpty(ca).Intersect(toBeRemovedIndex));
var dependsOnExisting = refreshBecauseNew.Intersect(refreshBecauseOld);
refreshes.AddRange(refreshBecauseNew.Except(refreshBecauseOld)
.Select(dependency => (refreshedConcept, newApplicationsByKey[dependency], RefreshDependencyStatus.New)));
protected static void AddConceptApplicationSeparator(ConceptApplication ca, CodeBuilder sqlCodeBuilder)
{
sqlCodeBuilder.InsertCode(string.Format("{0}{1}{2}{3}",
NextConceptApplicationSeparator, NextConceptApplicationIdPrefix, ca.Id, NextConceptApplicationIdSuffix));
}
