public async Task <ModuleLocation> ResolveInfoToLocationAsync( ModuleInfo moduleInfo, string originBucketName, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError, bool allowCaching = false ) { if (originBucketName == null) { throw new ArgumentNullException(nameof(originBucketName)); } LogInfoVerbose($"... resolving module {moduleInfo}"); var stopwatch = Stopwatch.StartNew(); var cached = false; try { // check if a cached manifest matches var cachedDirectory = Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo)); if (allowCaching && Settings.AllowCaching && Directory.Exists(cachedDirectory)) { var foundCached = Directory.GetFiles(cachedDirectory) .Select(found => VersionInfo.Parse(Path.GetFileName(found))) .Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)); // NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null)) { foundCached = foundCached.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(foundCached, moduleInfo.Version, candidate => { var candidateManifestText = File.ReadAllText(Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo), candidate.ToString())); manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText); // check if module is compatible with this tool return(VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(manifest.CoreServicesVersion, Settings.ToolVersion)); }); if (manifest != null) { cached = true; // TODO (2019-10-08, bjorg): what source bucket name should be used for cached manifests? return(MakeModuleLocation(Settings.DeploymentBucketName, manifest)); } } // check if module can be found in the deployment bucket var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName); // check if the origin bucket needs to be checked if ( allowImport && (Settings.DeploymentBucketName != originBucketName) && ( // no version has been found (result.Version == null) // no module version constraint was given; the ultimate floating version || (moduleInfo.Version == null) // the module version constraint is for a pre-release; we always prefer the origin version then || moduleInfo.Version.IsPreRelease() // the module version constraint is floating; we need to check if origin has a newer version || !moduleInfo.Version.Minor.HasValue || !moduleInfo.Version.Patch.HasValue ) ) { var originResult = await FindNewestModuleVersionAsync(originBucketName); // check if module found at origin should be kept instead if ( (originResult.Version != null) && ( (result.Version == null) || (moduleInfo.Version?.IsPreRelease() ?? false) || originResult.Version.IsGreaterThanVersion(result.Version) ) ) { result = originResult; } } // check if a module was found if (result.Version == null) { // could not find a matching version var versionConstraint = (moduleInfo.Version != null) ? $"v{moduleInfo.Version} or later" : "any released version"; if (showError) { if (allowImport) { LogError($"could not find module '{moduleInfo}' ({versionConstraint})"); } else { LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})"); } } return(null); } LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}"); // cache found version Directory.CreateDirectory(cachedDirectory); await File.WriteAllTextAsync(Path.Combine(cachedDirectory, result.Version.ToString()), JsonConvert.SerializeObject(result.Manifest)); return(MakeModuleLocation(result.Origin, result.Manifest)); } finally { LogInfoPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}", stopwatch.Elapsed, cached); } async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName) { // enumerate versions in bucket var found = await FindModuleVersionsAsync(bucketName); if (!found.Any()) { return(Origin : bucketName, Version : null, Manifest : null); } // NOTE (2019-08-12, bjorg): if the module is nested, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType == ModuleManifestDependencyType.Nested) && (moduleInfo.Version != null)) { found = found.Where(version => { if (!version.MatchesConstraint(moduleInfo.Version)) { LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}"); return(false); } return(true); }).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => { var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin); // check if the module version is allowed by the build policy if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true)) { LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy"); return(false); } // check if module is compatible with this tool var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).GetAwaiter().GetResult(); manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText); if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(manifest.CoreServicesVersion, Settings.ToolVersion)) { LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}"); return(false); } return(true); }); return(Origin : bucketName, Version : match, Manifest : manifest); } async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName) { // get bucket region specific S3 client var s3Client = await GetS3ClientByBucketNameAsync(bucketName); if (s3Client == null) { // nothing to do; GetS3ClientByBucketName already emitted an error return(new List <VersionInfo>()); } // enumerate versions in bucket var versions = new List <VersionInfo>(); var request = new ListObjectsV2Request { BucketName = bucketName, Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/", Delimiter = "/", MaxKeys = 100, RequestPayer = RequestPayer.Requester }; do { try { var response = await s3Client.ListObjectsV2Async(request); versions.AddRange(response.S3Objects .Select(s3Object => s3Object.Key.Substring(request.Prefix.Length)) .Select(found => VersionInfo.Parse(found)) .Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)) ); request.ContinuationToken = response.NextContinuationToken; } catch (AmazonS3Exception e) when(e.Message == "Access Denied") { // show message that access was denied for this location LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(versions); } } while(request.ContinuationToken != null); LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(versions); } ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest) => new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum); }
public async Task <ModuleLocation> ResolveInfoToLocationAsync( ModuleInfo moduleInfo, string bucketName, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError ) { if (bucketName == null) { throw new ArgumentNullException(nameof(bucketName)); } LogInfoVerbose($"... resolving module {moduleInfo}"); StartLogPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}"); var cached = false; try { // check if module can be found in the deployment bucket var result = await FindNewestModuleVersionInBucketAsync(Settings.DeploymentBucketName); // check if the origin bucket needs to be checked if ( allowImport && (Settings.DeploymentBucketName != bucketName) && ( // no version has been found (result.Version == null) // no module version constraint was given; the ultimate floating version || (moduleInfo.Version == null) // the module version constraint is for a pre-release; we always prefer the origin version then || moduleInfo.Version.IsPreRelease() // the module version constraint is floating; we need to check if origin has a newer version || !moduleInfo.Version.Minor.HasValue || !moduleInfo.Version.Patch.HasValue ) ) { var originResult = await FindNewestModuleVersionInBucketAsync(bucketName); // check if module found at origin should be kept instead if ( (originResult.Version != null) && ( (result.Version == null) || (moduleInfo.Version?.IsPreRelease() ?? false) || originResult.Version.IsGreaterThanVersion(result.Version) ) ) { result = originResult; } } // check if a module was found if (result.Version == null) { // could not find a matching version var versionConstraint = (moduleInfo.Version != null) ? $"v{moduleInfo.Version} or later" : "any released version"; if (showError) { if (allowImport) { LogError($"could not find module '{moduleInfo}' ({versionConstraint})"); } else { LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})"); } } return(null); } LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}"); return(MakeModuleLocation(result.Origin, result.Manifest)); } finally { StopLogPerformance(cached); } async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionInBucketAsync(string bucketName) { StartLogPerformance($"FindNewestModuleVersionInBucketAsync() for s3://{bucketName}"); try { // enumerate versions in bucket var found = await FindModuleVersionsInBucketAsync(bucketName); if (!found.Any()) { return(Origin : bucketName, Version : null, Manifest : null); } // NOTE (2019-08-12, bjorg): if the module is nested, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType == ModuleManifestDependencyType.Nested) && (moduleInfo.Version != null)) { found = found.Where(version => { if (!version.MatchesConstraint(moduleInfo.Version)) { LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}"); return(false); } return(true); }).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => { var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin); // check if the module version is allowed by the build policy if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true)) { LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy"); return(false); } // load module manifest var(candidateManifest, candidateManifestErrorReason) = LoadManifestFromLocationAsync(new ModuleLocation(bucketName, candidateModuleInfo, "<MISSING>")).GetAwaiter().GetResult(); if (candidateManifest == null) { LogInfoVerbose($"... rejected v{candidateVersion}: {candidateManifestErrorReason}"); return(false); } // check if module is compatible with this tool if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(candidateManifest.CoreServicesVersion, Settings.ToolVersion)) { LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}"); return(false); } // keep this manifest manifest = candidateManifest; return(true); }); return(Origin : bucketName, Version : match, Manifest : manifest); } finally { StopLogPerformance(); } } async Task <IEnumerable <VersionInfo> > FindModuleVersionsInBucketAsync(string bucketName) { StartLogPerformance($"FindModuleVersionsInBucketAsync() for s3://{bucketName}"); var cached = false; try { var moduleOrigin = moduleInfo.Origin ?? Settings.DeploymentBucketName; List <VersionInfo> versions = null; string region = null; // check if a cached version exists string cachedManifestVersionsFilePath = null; if (!Settings.ForceRefresh) { var cachedManifestFolder = GetCachedManifestDirectory(bucketName, moduleOrigin, moduleInfo.Namespace, moduleInfo.Name); if (cachedManifestFolder != null) { cachedManifestVersionsFilePath = Path.Combine(cachedManifestFolder, "versions.json"); if ( File.Exists(cachedManifestVersionsFilePath) && (File.GetLastWriteTimeUtc(cachedManifestVersionsFilePath).Add(Settings.CachedManifestListingExpiration) > DateTime.UtcNow) ) { cached = true; var cachedManifestVersions = JsonSerializer.Deserialize <ModuleManifestVersions>(File.ReadAllText(cachedManifestVersionsFilePath), Settings.JsonSerializerOptions); region = cachedManifestVersions.Region; versions = cachedManifestVersions.Versions; } } } // check if data needs to be fetched from S3 bucket if (versions == null) { // get bucket region specific S3 client var s3Client = await GetS3ClientByBucketNameAsync(bucketName); if (s3Client == null) { // nothing to do; GetS3ClientByBucketName already emitted an error return(new List <VersionInfo>()); } // enumerate versions in bucket versions = new List <VersionInfo>(); region = s3Client.Config.RegionEndpoint.SystemName; var request = new ListObjectsV2Request { BucketName = bucketName, Prefix = $"{moduleOrigin}/{moduleInfo.Namespace}/{moduleInfo.Name}/", Delimiter = "/", MaxKeys = 100, RequestPayer = RequestPayer.Requester }; do { try { var response = await s3Client.ListObjectsV2Async(request); versions.AddRange(response.S3Objects .Select(s3Object => s3Object.Key.Substring(request.Prefix.Length)) .Select(found => VersionInfo.Parse(found)) ); request.ContinuationToken = response.NextContinuationToken; } catch (AmazonS3Exception e) when(e.Message == "Access Denied") { // show message that access was denied for this location LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(Enumerable.Empty <VersionInfo>()); } } while(request.ContinuationToken != null); // cache module versions listing if (cachedManifestVersionsFilePath != null) { try { File.WriteAllText(cachedManifestVersionsFilePath, JsonSerializer.Serialize(new ModuleManifestVersions { Region = region, Versions = versions }, Settings.JsonSerializerOptions)); } catch { // nothing to do } } } // filter list down to matching versions versions = versions.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)).ToList(); LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{region}]"); return(versions); } finally { StopLogPerformance(cached); } } ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest) => new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum); }
public Task <ModuleLocation> ResolveInfoToLocationAsync(ModuleInfo moduleInfo, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError, bool allowCaching = false) => ResolveInfoToLocationAsync(moduleInfo, moduleInfo.Origin, dependencyType, allowImport, showError, allowCaching);
public async Task <ModuleLocation> ResolveInfoToLocationAsync(ModuleInfo moduleInfo, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError) { LogInfoVerbose($"=> Resolving module {moduleInfo}"); // check if module can be found in the deployment bucket var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName); // check if the origin bucket needs to be checked if ( allowImport && (Settings.DeploymentBucketName != moduleInfo.Origin) && ( // no version has been found (result.Version == null) // no module version constraint was given; the ultimate floating version || (moduleInfo.Version == null) // the module version constraint is for a pre-release; we always prefer the origin version then || moduleInfo.Version.IsPreRelease // the module version constraint is floating; we need to check if origin has a newer version || moduleInfo.Version.HasFloatingConstraints ) ) { var originResult = await FindNewestModuleVersionAsync(moduleInfo.Origin); // check if module found at origin should be kept instead if ( (originResult.Version != null) && ( (result.Version == null) || (moduleInfo.Version?.IsPreRelease ?? false) || originResult.Version.IsGreaterThanVersion(result.Version) ) ) { result = originResult; } } // check if a module was found if (result.Version == null) { // could not find a matching version var versionConstraint = (moduleInfo.Version != null) ? $"v{moduleInfo.Version} or later" : "any version"; if (showError) { LogError($"could not find module '{moduleInfo}' ({versionConstraint})"); } return(null); } LogInfoVerbose($"=> Selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}"); return(MakeModuleLocation(result.Origin, result.Manifest)); // local functions async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName) { // enumerate versions in bucket var found = await FindModuleVersionsAsync(bucketName); if (!found.Any()) { return(Origin : bucketName, Version : null, Manifest : null); } // NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null)) { found = found.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidate => { var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidate, moduleInfo.Origin); var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).Result; manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText); // check if module is compatible with this tool return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.CoreServicesVersion)); }); return(Origin : bucketName, Version : match, Manifest : manifest); } async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName) { // get bucket region specific S3 client var s3Client = await GetS3ClientByBucketNameAsync(bucketName); if (s3Client == null) { // nothing to do; GetS3ClientByBucketName already emitted an error return(new List <VersionInfo>()); } // enumerate versions in bucket var versions = new List <VersionInfo>(); var request = new ListObjectsV2Request { BucketName = bucketName, Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/", Delimiter = "/", MaxKeys = 100, RequestPayer = RequestPayer.Requester }; do { try { var response = await s3Client.ListObjectsV2Async(request); versions.AddRange(response.S3Objects .Select(s3Object => s3Object.Key.Substring(request.Prefix.Length)) .Select(found => VersionInfo.Parse(found)) .Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)) ); request.ContinuationToken = response.NextContinuationToken; } catch (AmazonS3Exception e) when(e.Message == "Access Denied") { break; } } while(request.ContinuationToken != null); LogInfoVerbose($"==> Found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(versions); } ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest) => new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum); }