private async Task <IEnumerable <(ModuleManifest Manifest, ModuleLocation Location)> > DiscoverDependenciesAsync(ModuleManifest manifest)
{
var deployments = new List <DependencyRecord>();
var existing = new List <DependencyRecord>();
var inProgress = new List <DependencyRecord>();
// create a topological sort of dependencies
await Recurse(manifest);
return(deployments.Select(tuple => (tuple.Manifest, tuple.Location)).ToList());
// local functions
async Task Recurse(ModuleManifest current)
{
foreach (var dependency in current.Dependencies)
{
// check if we have already discovered this dependency
if (IsDependencyInList(current.GetFullName(), dependency, existing) || IsDependencyInList(current.GetFullName(), dependency, deployments))
{
continue;
}
// check if this dependency needs to be deployed
var deployed = await FindExistingDependencyAsync(dependency);
if (deployed != null)
{
existing.Add(new DependencyRecord {
Location = deployed
});
}
else if (inProgress.Any(d => d.Manifest.GetFullName() == dependency.ModuleFullName))
{
// circular dependency detected
LogError($"circular dependency detected: {string.Join(" -> ", inProgress.Select(d => d.Manifest.GetFullName()))}");
return;
}
else
{
dependency.ModuleFullName.TryParseModuleOwnerName(out string moduleOwner, out var moduleName);
// resolve dependencies for dependency module
var dependencyLocation = await _loader.LocateAsync(moduleOwner, moduleName, dependency.MinVersion, dependency.MaxVersion, dependency.BucketName);
if (dependencyLocation == null)
{
// error has already been reported
continue;
}
// load manifest of dependency and add its dependencies
var dependencyManifest = await _loader.LoadFromS3Async(dependencyLocation.ModuleBucketName, dependencyLocation.TemplatePath);
if (dependencyManifest == null)
{
// error has already been reported
continue;
}
var nestedDependency = new DependencyRecord {
Owner = current.Module,
Manifest = dependencyManifest,
Location = dependencyLocation
};
// keep marker for in-progress resolutions so that circular errors can be detected
inProgress.Add(nestedDependency);
await Recurse(dependencyManifest);
inProgress.Remove(nestedDependency);
// append dependency now that all nested dependencies have been resolved
Console.WriteLine($"=> Resolved dependency '{dependency.ModuleFullName}' to module reference: {dependencyLocation}");
deployments.Add(nestedDependency);
}
}
}
}
public async Task <IEnumerable <(ModuleManifest Manifest, ModuleLocation ModuleLocation, ModuleManifestDependencyType Type)> > DiscoverAllDependenciesAsync(
ModuleManifest manifest,
bool checkExisting,
bool allowImport,
bool allowDependencyUpgrades
)
{
var deployments = new List <DependencyRecord>();
var existing = new List <DependencyRecord>();
var inProgress = new List <DependencyRecord>();
// create a topological sort of dependencies
await Recurse(manifest);
return(deployments.Select(record => (record.Manifest, record.ModuleLocation, record.Type)).ToList());
// local functions
async Task Recurse(ModuleManifest current)
{
foreach (var dependency in current.Dependencies)
{
// check if we have already discovered this dependency
if (IsDependencyInList(current.GetFullName(), dependency, existing) || IsDependencyInList(current.GetFullName(), dependency, deployments))
{
continue;
}
// check if this dependency needs to be deployed
var existingDependency = checkExisting
? await FindExistingDependencyAsync(dependency)
: null;
if (existingDependency != null)
{
existing.Add(existingDependency);
}
else if (inProgress.Any(d => d.Manifest.ModuleInfo.FullName == dependency.ModuleInfo.FullName))
{
// circular dependency detected
LogError($"circular dependency detected: {string.Join(" -> ", inProgress.Select(d => d.Manifest.GetFullName()))}");
return;
}
else
{
// resolve dependencies for dependency module
var dependencyModuleLocation = await ResolveInfoToLocationAsync(dependency.ModuleInfo, dependency.Type, allowImport, showError : true);
if (dependencyModuleLocation == null)
{
// nothing to do; loader already emitted an error
continue;
}
// load manifest of dependency and add its dependencies
var dependencyManifest = await LoadManifestFromLocationAsync(dependencyModuleLocation);
if (dependencyManifest == null)
{
// error has already been reported
continue;
}
var nestedDependency = new DependencyRecord {
DependencyOwner = current.ModuleInfo,
Manifest = dependencyManifest,
ModuleLocation = dependencyModuleLocation,
Type = dependency.Type
};
// keep marker for in-progress resolutions so that circular errors can be detected
inProgress.Add(nestedDependency);
await Recurse(dependencyManifest);
inProgress.Remove(nestedDependency);
// append dependency now that all nested dependencies have been resolved
LogInfoVerbose($"... resolved dependency '{dependency.ModuleInfo.FullName}' to {dependencyModuleLocation.ModuleInfo}");
deployments.Add(nestedDependency);
}
}
}
bool IsDependencyInList(string dependentModuleFullName, ModuleManifestDependency dependency, IEnumerable <DependencyRecord> deployedModules)
{
var deployedModule = deployedModules.FirstOrDefault(deployed => deployed.ModuleLocation.ModuleInfo.FullName == dependency.ModuleInfo.FullName);
if (deployedModule == null)
{
return(false);
}
var deployedOwner = (deployedModule.DependencyOwner == null)
? "existing module"
: $"module '{deployedModule.DependencyOwner}'";
// confirm the requested version by the dependency is not greater than the deployed version
var deployedVersion = deployedModule.ModuleLocation.ModuleInfo.Version;
if (dependency.ModuleInfo.Version?.IsGreaterThanVersion(deployedVersion) ?? false)
{
LogError($"version conflict for module '{dependency.ModuleInfo.FullName}': module '{dependentModuleFullName}' requires v{dependency.ModuleInfo.Version}, but {deployedOwner} uses v{deployedVersion})");
}
return(true);
}
async Task <DependencyRecord> FindExistingDependencyAsync(ModuleManifestDependency dependency)
{
// attempt to find an existing, deployed stack matching the dependency
var stackName = Settings.GetStackName(dependency.ModuleInfo.FullName);
var deployedModule = await Settings.CfnClient.GetStackAsync(stackName, LogError);
if (deployedModule.Stack == null)
{
return(null);
}
if (!ModuleInfo.TryParse(deployedModule.Stack.GetModuleVersionText(), out var deployedModuleInfo))
{
LogWarn($"unable to retrieve module version from CloudFormation stack '{stackName}'");
return(null);
}
if (deployedModule.Stack.GetModuleManifestChecksum() == null)
{
LogWarn($"unable to retrieve module checksum from CloudFormation stack '{stackName}'");
return(null);
}
var result = new DependencyRecord {
ModuleLocation = new ModuleLocation(Settings.DeploymentBucketName, deployedModuleInfo, deployedModule.Stack.GetModuleManifestChecksum())
};
// confirm that the module name, version and hash match
if (deployedModuleInfo.FullName != dependency.ModuleInfo.FullName)
{
LogError($"deployed dependent module name ({deployedModuleInfo.FullName}) does not match {dependency.ModuleInfo.FullName}");
}
else if (!deployedModuleInfo.Version.MatchesConstraint(dependency.ModuleInfo.Version))
{
// for out-of-date dependencies, handle them as if they didn't exist when upgrades are allowed
if (allowDependencyUpgrades)
{
return(null);
}
LogError($"deployed dependent module {dependency.ModuleInfo.FullName} (v{deployedModuleInfo.Version}) is not compatible with v{dependency.ModuleInfo.Version}");
}
return(result);
}
}
public async Task<IDictionary<TypedDatabaseObject, GraphNode>> WalkDependenciesAsync()
{
Dictionary<TypedDatabaseObject, GraphNode> objects = new Dictionary<TypedDatabaseObject, GraphNode>(new TypedDatabaseObjectComparer());
using (var conn = Database.Connection())
{
await conn.OpenAsync();
SqlCommand cmd = new SqlCommand(@"
Select
ISNULL(o.name, t.name) as objectName,
s.name as schemaName,
Case
When sed.referencing_class = 6 Then 'TT'
Else o.[type]
End as objectType,
sed.referenced_entity_name as dependencyName,
ISNULL(sed.referenced_schema_name, 'dbo') as dependencySchemaName,
Case
When sed.referenced_class = 6 Then 'TT'
Else dep.[type]
End as dependencyType
From sys.sql_expression_dependencies sed
left outer join sys.objects o
on sed.referencing_id = o.[object_id]
and sed.referencing_class <> 6
and o.[type] in ('V', 'FN', 'IF', 'P')
left outer join sys.types t
on sed.referencing_id = t.user_type_id
and sed.referencing_class = 6
and t.is_user_defined = 1
inner join sys.schemas s
on ISNULL(o.[schema_id], t.[schema_id]) = s.[schema_id]
left join sys.objects dep
on sed.referenced_id = dep.[object_id]
and sed.referenced_class <> 6
and dep.[type] in ('V', 'FN', 'IF', 'P')
Where
ISNULL(o.name, t.name) is not null
and (
sed.referenced_class = 6
or dep.[type] is not null
)
and ISNULL(sed.referenced_database_name, DB_NAME()) = DB_NAME()
and ISNULL(sed.referenced_server_name, @@SERVERNAME) = @@SERVERNAME
Union
Select
o.name as objectName,
s.name as schemaName,
o.[type] as objectType,
dep.name as dependencyName,
deps.name as dependencySchemaName,
dep.[type] as dependencyType
From sys.sql_dependencies sd
inner join sys.objects o
on sd.[object_id] = o.[object_id]
inner join sys.schemas s
on o.[schema_id] = s.[schema_id]
inner join sys.objects dep
on sd.referenced_major_id = dep.[object_id]
inner join sys.schemas deps
on dep.[schema_id] = deps.[schema_id]
Where
o.[type] in ('V', 'FN', 'IF', 'P')
and dep.[type] in ('V', 'FN', 'IF', 'P')
", conn);
using(var reader = await cmd.ExecuteReaderAsync())
{
while(await reader.ReadAsync())
{
var record = new DependencyRecord()
{
ObjectName = reader["objectName"] as string,
SchemaName = reader["schemaName"] as string,
Type = reader["objectType"] as string,
DependencyName = reader["dependencyName"] as string,
DependencySchemaName = reader["dependencySchemaName"] as string,
DependencyType = reader["dependencyType"] as string
};
var dependant = record.GetObject(Database.ServerName, Database.DatabaseName);
var dependency = record.GetDependency(Database.ServerName, Database.DatabaseName);
if(!objects.ContainsKey(dependant))
{
objects.Add(dependant, new GraphNode(dependant));
}
if(!objects.ContainsKey(dependency))
{
objects.Add(dependency, new GraphNode(dependency));
}
objects[dependant].Dependencies.Add(dependency);
objects[dependency].ReferencedBy.Add(dependant);
}
}
}
return objects;
}
