private async Task <IEnumerable <(ModuleManifest Manifest, ModuleLocation Location)> > DiscoverDependenciesAsync(ModuleManifest manifest) { var deployments = new List <DependencyRecord>(); var existing = new List <DependencyRecord>(); var inProgress = new List <DependencyRecord>(); // create a topological sort of dependencies await Recurse(manifest); return(deployments.Select(tuple => (tuple.Manifest, tuple.Location)).ToList()); // local functions async Task Recurse(ModuleManifest current) { foreach (var dependency in current.Dependencies) { // check if we have already discovered this dependency if (IsDependencyInList(current.GetFullName(), dependency, existing) || IsDependencyInList(current.GetFullName(), dependency, deployments)) { continue; } // check if this dependency needs to be deployed var deployed = await FindExistingDependencyAsync(dependency); if (deployed != null) { existing.Add(new DependencyRecord { Location = deployed }); } else if (inProgress.Any(d => d.Manifest.GetFullName() == dependency.ModuleFullName)) { // circular dependency detected LogError($"circular dependency detected: {string.Join(" -> ", inProgress.Select(d => d.Manifest.GetFullName()))}"); return; } else { dependency.ModuleFullName.TryParseModuleOwnerName(out string moduleOwner, out var moduleName); // resolve dependencies for dependency module var dependencyLocation = await _loader.LocateAsync(moduleOwner, moduleName, dependency.MinVersion, dependency.MaxVersion, dependency.BucketName); if (dependencyLocation == null) { // error has already been reported continue; } // load manifest of dependency and add its dependencies var dependencyManifest = await _loader.LoadFromS3Async(dependencyLocation.ModuleBucketName, dependencyLocation.TemplatePath); if (dependencyManifest == null) { // error has already been reported continue; } var nestedDependency = new DependencyRecord { Owner = current.Module, Manifest = dependencyManifest, Location = dependencyLocation }; // keep marker for in-progress resolutions so that circular errors can be detected inProgress.Add(nestedDependency); await Recurse(dependencyManifest); inProgress.Remove(nestedDependency); // append dependency now that all nested dependencies have been resolved Console.WriteLine($"=> Resolved dependency '{dependency.ModuleFullName}' to module reference: {dependencyLocation}"); deployments.Add(nestedDependency); } } } }
public async Task <IEnumerable <(ModuleManifest Manifest, ModuleLocation ModuleLocation, ModuleManifestDependencyType Type)> > DiscoverAllDependenciesAsync( ModuleManifest manifest, bool checkExisting, bool allowImport, bool allowDependencyUpgrades ) { var deployments = new List <DependencyRecord>(); var existing = new List <DependencyRecord>(); var inProgress = new List <DependencyRecord>(); // create a topological sort of dependencies await Recurse(manifest); return(deployments.Select(record => (record.Manifest, record.ModuleLocation, record.Type)).ToList()); // local functions async Task Recurse(ModuleManifest current) { foreach (var dependency in current.Dependencies) { // check if we have already discovered this dependency if (IsDependencyInList(current.GetFullName(), dependency, existing) || IsDependencyInList(current.GetFullName(), dependency, deployments)) { continue; } // check if this dependency needs to be deployed var existingDependency = checkExisting ? await FindExistingDependencyAsync(dependency) : null; if (existingDependency != null) { existing.Add(existingDependency); } else if (inProgress.Any(d => d.Manifest.ModuleInfo.FullName == dependency.ModuleInfo.FullName)) { // circular dependency detected LogError($"circular dependency detected: {string.Join(" -> ", inProgress.Select(d => d.Manifest.GetFullName()))}"); return; } else { // resolve dependencies for dependency module var dependencyModuleLocation = await ResolveInfoToLocationAsync(dependency.ModuleInfo, dependency.Type, allowImport, showError : true); if (dependencyModuleLocation == null) { // nothing to do; loader already emitted an error continue; } // load manifest of dependency and add its dependencies var dependencyManifest = await LoadManifestFromLocationAsync(dependencyModuleLocation); if (dependencyManifest == null) { // error has already been reported continue; } var nestedDependency = new DependencyRecord { DependencyOwner = current.ModuleInfo, Manifest = dependencyManifest, ModuleLocation = dependencyModuleLocation, Type = dependency.Type }; // keep marker for in-progress resolutions so that circular errors can be detected inProgress.Add(nestedDependency); await Recurse(dependencyManifest); inProgress.Remove(nestedDependency); // append dependency now that all nested dependencies have been resolved LogInfoVerbose($"... resolved dependency '{dependency.ModuleInfo.FullName}' to {dependencyModuleLocation.ModuleInfo}"); deployments.Add(nestedDependency); } } } bool IsDependencyInList(string dependentModuleFullName, ModuleManifestDependency dependency, IEnumerable <DependencyRecord> deployedModules) { var deployedModule = deployedModules.FirstOrDefault(deployed => deployed.ModuleLocation.ModuleInfo.FullName == dependency.ModuleInfo.FullName); if (deployedModule == null) { return(false); } var deployedOwner = (deployedModule.DependencyOwner == null) ? "existing module" : $"module '{deployedModule.DependencyOwner}'"; // confirm the requested version by the dependency is not greater than the deployed version var deployedVersion = deployedModule.ModuleLocation.ModuleInfo.Version; if (dependency.ModuleInfo.Version?.IsGreaterThanVersion(deployedVersion) ?? false) { LogError($"version conflict for module '{dependency.ModuleInfo.FullName}': module '{dependentModuleFullName}' requires v{dependency.ModuleInfo.Version}, but {deployedOwner} uses v{deployedVersion})"); } return(true); } async Task <DependencyRecord> FindExistingDependencyAsync(ModuleManifestDependency dependency) { // attempt to find an existing, deployed stack matching the dependency var stackName = Settings.GetStackName(dependency.ModuleInfo.FullName); var deployedModule = await Settings.CfnClient.GetStackAsync(stackName, LogError); if (deployedModule.Stack == null) { return(null); } if (!ModuleInfo.TryParse(deployedModule.Stack.GetModuleVersionText(), out var deployedModuleInfo)) { LogWarn($"unable to retrieve module version from CloudFormation stack '{stackName}'"); return(null); } if (deployedModule.Stack.GetModuleManifestChecksum() == null) { LogWarn($"unable to retrieve module checksum from CloudFormation stack '{stackName}'"); return(null); } var result = new DependencyRecord { ModuleLocation = new ModuleLocation(Settings.DeploymentBucketName, deployedModuleInfo, deployedModule.Stack.GetModuleManifestChecksum()) }; // confirm that the module name, version and hash match if (deployedModuleInfo.FullName != dependency.ModuleInfo.FullName) { LogError($"deployed dependent module name ({deployedModuleInfo.FullName}) does not match {dependency.ModuleInfo.FullName}"); } else if (!deployedModuleInfo.Version.MatchesConstraint(dependency.ModuleInfo.Version)) { // for out-of-date dependencies, handle them as if they didn't exist when upgrades are allowed if (allowDependencyUpgrades) { return(null); } LogError($"deployed dependent module {dependency.ModuleInfo.FullName} (v{deployedModuleInfo.Version}) is not compatible with v{dependency.ModuleInfo.Version}"); } return(result); } }
public async Task<IDictionary<TypedDatabaseObject, GraphNode>> WalkDependenciesAsync() { Dictionary<TypedDatabaseObject, GraphNode> objects = new Dictionary<TypedDatabaseObject, GraphNode>(new TypedDatabaseObjectComparer()); using (var conn = Database.Connection()) { await conn.OpenAsync(); SqlCommand cmd = new SqlCommand(@" Select ISNULL(o.name, t.name) as objectName, s.name as schemaName, Case When sed.referencing_class = 6 Then 'TT' Else o.[type] End as objectType, sed.referenced_entity_name as dependencyName, ISNULL(sed.referenced_schema_name, 'dbo') as dependencySchemaName, Case When sed.referenced_class = 6 Then 'TT' Else dep.[type] End as dependencyType From sys.sql_expression_dependencies sed left outer join sys.objects o on sed.referencing_id = o.[object_id] and sed.referencing_class <> 6 and o.[type] in ('V', 'FN', 'IF', 'P') left outer join sys.types t on sed.referencing_id = t.user_type_id and sed.referencing_class = 6 and t.is_user_defined = 1 inner join sys.schemas s on ISNULL(o.[schema_id], t.[schema_id]) = s.[schema_id] left join sys.objects dep on sed.referenced_id = dep.[object_id] and sed.referenced_class <> 6 and dep.[type] in ('V', 'FN', 'IF', 'P') Where ISNULL(o.name, t.name) is not null and ( sed.referenced_class = 6 or dep.[type] is not null ) and ISNULL(sed.referenced_database_name, DB_NAME()) = DB_NAME() and ISNULL(sed.referenced_server_name, @@SERVERNAME) = @@SERVERNAME Union Select o.name as objectName, s.name as schemaName, o.[type] as objectType, dep.name as dependencyName, deps.name as dependencySchemaName, dep.[type] as dependencyType From sys.sql_dependencies sd inner join sys.objects o on sd.[object_id] = o.[object_id] inner join sys.schemas s on o.[schema_id] = s.[schema_id] inner join sys.objects dep on sd.referenced_major_id = dep.[object_id] inner join sys.schemas deps on dep.[schema_id] = deps.[schema_id] Where o.[type] in ('V', 'FN', 'IF', 'P') and dep.[type] in ('V', 'FN', 'IF', 'P') ", conn); using(var reader = await cmd.ExecuteReaderAsync()) { while(await reader.ReadAsync()) { var record = new DependencyRecord() { ObjectName = reader["objectName"] as string, SchemaName = reader["schemaName"] as string, Type = reader["objectType"] as string, DependencyName = reader["dependencyName"] as string, DependencySchemaName = reader["dependencySchemaName"] as string, DependencyType = reader["dependencyType"] as string }; var dependant = record.GetObject(Database.ServerName, Database.DatabaseName); var dependency = record.GetDependency(Database.ServerName, Database.DatabaseName); if(!objects.ContainsKey(dependant)) { objects.Add(dependant, new GraphNode(dependant)); } if(!objects.ContainsKey(dependency)) { objects.Add(dependency, new GraphNode(dependency)); } objects[dependant].Dependencies.Add(dependency); objects[dependency].ReferencedBy.Add(dependant); } } } return objects; }