private async Task <VersionInfo> FindNewestVersion(Settings settings, string bucketName, string moduleOwner, string moduleName, VersionInfo minVersion, VersionInfo maxVersion)
{
// enumerate versions in bucket
var versions = new List <VersionInfo>();
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleOwner}/Modules/{moduleName}/Versions/",
Delimiter = "/",
MaxKeys = 100
};
do
{
var response = await settings.S3Client.ListObjectsV2Async(request);
versions.AddRange(response.CommonPrefixes
.Select(prefix => prefix.Substring(request.Prefix.Length).TrimEnd('/'))
.Select(found => VersionInfo.Parse(found))
.Where(IsVersionMatch)
);
request.ContinuationToken = response.NextContinuationToken;
} while(request.ContinuationToken != null);
if (!versions.Any())
{
return(null);
}
// attempt to identify the newest version
return(versions.Max());
// local function
bool IsVersionMatch(VersionInfo version)
{
if ((minVersion == null) && (maxVersion == null))
{
return(!version.IsPreRelease);
}
if (maxVersion == minVersion)
{
return(version.IsCompatibleWith(minVersion));
}
if ((minVersion != null) && (version < minVersion))
{
return(false);
}
if ((maxVersion != null) && (version > maxVersion))
{
return(false);
}
return(true);
}
}
public static bool TryParse(string moduleReference, out ModuleInfo moduleInfo)
{
if (moduleReference == null)
{
moduleInfo = null;
return(false);
}
// try parsing module reference
var match = ModuleKeyPattern.Match(moduleReference);
if (!match.Success)
{
moduleInfo = null;
return(false);
}
var ns = GetMatchValue("Namespace");
var name = GetMatchValue("Name");
var origin = GetMatchValue("Origin");
// parse optional version
var versionText = GetMatchValue("Version");
var version = ((versionText != null) && (versionText != "*"))
? VersionInfo.Parse(versionText)
: null;
moduleInfo = new ModuleInfo(ns, name, version, origin);
return(true);
// local function
string GetMatchValue(string groupName)
{
var group = match.Groups[groupName];
return(group.Success ? group.Value : null);
}
}
public async Task <ModuleLocation> ResolveInfoToLocationAsync(
ModuleInfo moduleInfo,
string originBucketName,
ModuleManifestDependencyType dependencyType,
bool allowImport,
bool showError,
bool allowCaching = false
)
{
if (originBucketName == null)
{
throw new ArgumentNullException(nameof(originBucketName));
}
LogInfoVerbose($"... resolving module {moduleInfo}");
var stopwatch = Stopwatch.StartNew();
var cached = false;
try {
// check if a cached manifest matches
var cachedDirectory = Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo));
if (allowCaching && Settings.AllowCaching && Directory.Exists(cachedDirectory))
{
var foundCached = Directory.GetFiles(cachedDirectory)
.Select(found => VersionInfo.Parse(Path.GetFileName(found)))
.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true));
// NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null))
{
foundCached = foundCached.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(foundCached, moduleInfo.Version, candidate => {
var candidateManifestText = File.ReadAllText(Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo), candidate.ToString()));
manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText);
// check if module is compatible with this tool
return(VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(manifest.CoreServicesVersion, Settings.ToolVersion));
});
if (manifest != null)
{
cached = true;
// TODO (2019-10-08, bjorg): what source bucket name should be used for cached manifests?
return(MakeModuleLocation(Settings.DeploymentBucketName, manifest));
}
}
// check if module can be found in the deployment bucket
var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName);
// check if the origin bucket needs to be checked
if (
allowImport &&
(Settings.DeploymentBucketName != originBucketName) &&
(
// no version has been found
(result.Version == null)
// no module version constraint was given; the ultimate floating version
|| (moduleInfo.Version == null)
// the module version constraint is for a pre-release; we always prefer the origin version then
|| moduleInfo.Version.IsPreRelease()
// the module version constraint is floating; we need to check if origin has a newer version
|| !moduleInfo.Version.Minor.HasValue ||
!moduleInfo.Version.Patch.HasValue
)
)
{
var originResult = await FindNewestModuleVersionAsync(originBucketName);
// check if module found at origin should be kept instead
if (
(originResult.Version != null) &&
(
(result.Version == null) ||
(moduleInfo.Version?.IsPreRelease() ?? false) ||
originResult.Version.IsGreaterThanVersion(result.Version)
)
)
{
result = originResult;
}
}
// check if a module was found
if (result.Version == null)
{
// could not find a matching version
var versionConstraint = (moduleInfo.Version != null)
? $"v{moduleInfo.Version} or later"
: "any released version";
if (showError)
{
if (allowImport)
{
LogError($"could not find module '{moduleInfo}' ({versionConstraint})");
}
else
{
LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})");
}
}
return(null);
}
LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}");
// cache found version
Directory.CreateDirectory(cachedDirectory);
await File.WriteAllTextAsync(Path.Combine(cachedDirectory, result.Version.ToString()), JsonConvert.SerializeObject(result.Manifest));
return(MakeModuleLocation(result.Origin, result.Manifest));
} finally {
LogInfoPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}", stopwatch.Elapsed, cached);
}
async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName)
{
// enumerate versions in bucket
var found = await FindModuleVersionsAsync(bucketName);
if (!found.Any())
{
return(Origin : bucketName, Version : null, Manifest : null);
}
// NOTE (2019-08-12, bjorg): if the module is nested, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType == ModuleManifestDependencyType.Nested) && (moduleInfo.Version != null))
{
found = found.Where(version => {
if (!version.MatchesConstraint(moduleInfo.Version))
{
LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}");
return(false);
}
return(true);
}).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => {
var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin);
// check if the module version is allowed by the build policy
if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy");
return(false);
}
// check if module is compatible with this tool
var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).GetAwaiter().GetResult();
manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText);
if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(manifest.CoreServicesVersion, Settings.ToolVersion))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}");
return(false);
}
return(true);
});
return(Origin : bucketName, Version : match, Manifest : manifest);
}
async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName)
{
// get bucket region specific S3 client
var s3Client = await GetS3ClientByBucketNameAsync(bucketName);
if (s3Client == null)
{
// nothing to do; GetS3ClientByBucketName already emitted an error
return(new List <VersionInfo>());
}
// enumerate versions in bucket
var versions = new List <VersionInfo>();
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/",
Delimiter = "/",
MaxKeys = 100,
RequestPayer = RequestPayer.Requester
};
do
{
try {
var response = await s3Client.ListObjectsV2Async(request);
versions.AddRange(response.S3Objects
.Select(s3Object => s3Object.Key.Substring(request.Prefix.Length))
.Select(found => VersionInfo.Parse(found))
.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true))
);
request.ContinuationToken = response.NextContinuationToken;
} catch (AmazonS3Exception e) when(e.Message == "Access Denied")
{
// show message that access was denied for this location
LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(versions);
}
} while(request.ContinuationToken != null);
LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(versions);
}
ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest)
=> new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
=> (reader.Value != null)
? VersionInfo.Parse((string)reader.Value)
: null;
public async Task <ModuleLocation> ResolveInfoToLocationAsync(
ModuleInfo moduleInfo,
string bucketName,
ModuleManifestDependencyType dependencyType,
bool allowImport,
bool showError
)
{
if (bucketName == null)
{
throw new ArgumentNullException(nameof(bucketName));
}
LogInfoVerbose($"... resolving module {moduleInfo}");
StartLogPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}");
var cached = false;
try {
// check if module can be found in the deployment bucket
var result = await FindNewestModuleVersionInBucketAsync(Settings.DeploymentBucketName);
// check if the origin bucket needs to be checked
if (
allowImport &&
(Settings.DeploymentBucketName != bucketName) &&
(
// no version has been found
(result.Version == null)
// no module version constraint was given; the ultimate floating version
|| (moduleInfo.Version == null)
// the module version constraint is for a pre-release; we always prefer the origin version then
|| moduleInfo.Version.IsPreRelease()
// the module version constraint is floating; we need to check if origin has a newer version
|| !moduleInfo.Version.Minor.HasValue ||
!moduleInfo.Version.Patch.HasValue
)
)
{
var originResult = await FindNewestModuleVersionInBucketAsync(bucketName);
// check if module found at origin should be kept instead
if (
(originResult.Version != null) &&
(
(result.Version == null) ||
(moduleInfo.Version?.IsPreRelease() ?? false) ||
originResult.Version.IsGreaterThanVersion(result.Version)
)
)
{
result = originResult;
}
}
// check if a module was found
if (result.Version == null)
{
// could not find a matching version
var versionConstraint = (moduleInfo.Version != null)
? $"v{moduleInfo.Version} or later"
: "any released version";
if (showError)
{
if (allowImport)
{
LogError($"could not find module '{moduleInfo}' ({versionConstraint})");
}
else
{
LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})");
}
}
return(null);
}
LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}");
return(MakeModuleLocation(result.Origin, result.Manifest));
} finally {
StopLogPerformance(cached);
}
async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionInBucketAsync(string bucketName)
{
StartLogPerformance($"FindNewestModuleVersionInBucketAsync() for s3://{bucketName}");
try {
// enumerate versions in bucket
var found = await FindModuleVersionsInBucketAsync(bucketName);
if (!found.Any())
{
return(Origin : bucketName, Version : null, Manifest : null);
}
// NOTE (2019-08-12, bjorg): if the module is nested, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType == ModuleManifestDependencyType.Nested) && (moduleInfo.Version != null))
{
found = found.Where(version => {
if (!version.MatchesConstraint(moduleInfo.Version))
{
LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}");
return(false);
}
return(true);
}).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => {
var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin);
// check if the module version is allowed by the build policy
if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy");
return(false);
}
// load module manifest
var(candidateManifest, candidateManifestErrorReason) = LoadManifestFromLocationAsync(new ModuleLocation(bucketName, candidateModuleInfo, "<MISSING>")).GetAwaiter().GetResult();
if (candidateManifest == null)
{
LogInfoVerbose($"... rejected v{candidateVersion}: {candidateManifestErrorReason}");
return(false);
}
// check if module is compatible with this tool
if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(candidateManifest.CoreServicesVersion, Settings.ToolVersion))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}");
return(false);
}
// keep this manifest
manifest = candidateManifest;
return(true);
});
return(Origin : bucketName, Version : match, Manifest : manifest);
} finally {
StopLogPerformance();
}
}
async Task <IEnumerable <VersionInfo> > FindModuleVersionsInBucketAsync(string bucketName)
{
StartLogPerformance($"FindModuleVersionsInBucketAsync() for s3://{bucketName}");
var cached = false;
try {
var moduleOrigin = moduleInfo.Origin ?? Settings.DeploymentBucketName;
List <VersionInfo> versions = null;
string region = null;
// check if a cached version exists
string cachedManifestVersionsFilePath = null;
if (!Settings.ForceRefresh)
{
var cachedManifestFolder = GetCachedManifestDirectory(bucketName, moduleOrigin, moduleInfo.Namespace, moduleInfo.Name);
if (cachedManifestFolder != null)
{
cachedManifestVersionsFilePath = Path.Combine(cachedManifestFolder, "versions.json");
if (
File.Exists(cachedManifestVersionsFilePath) &&
(File.GetLastWriteTimeUtc(cachedManifestVersionsFilePath).Add(Settings.CachedManifestListingExpiration) > DateTime.UtcNow)
)
{
cached = true;
var cachedManifestVersions = JsonSerializer.Deserialize <ModuleManifestVersions>(File.ReadAllText(cachedManifestVersionsFilePath), Settings.JsonSerializerOptions);
region = cachedManifestVersions.Region;
versions = cachedManifestVersions.Versions;
}
}
}
// check if data needs to be fetched from S3 bucket
if (versions == null)
{
// get bucket region specific S3 client
var s3Client = await GetS3ClientByBucketNameAsync(bucketName);
if (s3Client == null)
{
// nothing to do; GetS3ClientByBucketName already emitted an error
return(new List <VersionInfo>());
}
// enumerate versions in bucket
versions = new List <VersionInfo>();
region = s3Client.Config.RegionEndpoint.SystemName;
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleOrigin}/{moduleInfo.Namespace}/{moduleInfo.Name}/",
Delimiter = "/",
MaxKeys = 100,
RequestPayer = RequestPayer.Requester
};
do
{
try {
var response = await s3Client.ListObjectsV2Async(request);
versions.AddRange(response.S3Objects
.Select(s3Object => s3Object.Key.Substring(request.Prefix.Length))
.Select(found => VersionInfo.Parse(found))
);
request.ContinuationToken = response.NextContinuationToken;
} catch (AmazonS3Exception e) when(e.Message == "Access Denied")
{
// show message that access was denied for this location
LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(Enumerable.Empty <VersionInfo>());
}
} while(request.ContinuationToken != null);
// cache module versions listing
if (cachedManifestVersionsFilePath != null)
{
try {
File.WriteAllText(cachedManifestVersionsFilePath, JsonSerializer.Serialize(new ModuleManifestVersions {
Region = region,
Versions = versions
}, Settings.JsonSerializerOptions));
} catch {
// nothing to do
}
}
}
// filter list down to matching versions
versions = versions.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)).ToList();
LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{region}]");
return(versions);
} finally {
StopLogPerformance(cached);
}
}
ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest)
=> new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum);
}
public async Task <ModuleLocation> ResolveInfoToLocationAsync(ModuleInfo moduleInfo, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError)
{
LogInfoVerbose($"=> Resolving module {moduleInfo}");
// check if module can be found in the deployment bucket
var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName);
// check if the origin bucket needs to be checked
if (
allowImport &&
(Settings.DeploymentBucketName != moduleInfo.Origin) &&
(
// no version has been found
(result.Version == null)
// no module version constraint was given; the ultimate floating version
|| (moduleInfo.Version == null)
// the module version constraint is for a pre-release; we always prefer the origin version then
|| moduleInfo.Version.IsPreRelease
// the module version constraint is floating; we need to check if origin has a newer version
|| moduleInfo.Version.HasFloatingConstraints
)
)
{
var originResult = await FindNewestModuleVersionAsync(moduleInfo.Origin);
// check if module found at origin should be kept instead
if (
(originResult.Version != null) &&
(
(result.Version == null) ||
(moduleInfo.Version?.IsPreRelease ?? false) ||
originResult.Version.IsGreaterThanVersion(result.Version)
)
)
{
result = originResult;
}
}
// check if a module was found
if (result.Version == null)
{
// could not find a matching version
var versionConstraint = (moduleInfo.Version != null)
? $"v{moduleInfo.Version} or later"
: "any version";
if (showError)
{
LogError($"could not find module '{moduleInfo}' ({versionConstraint})");
}
return(null);
}
LogInfoVerbose($"=> Selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}");
return(MakeModuleLocation(result.Origin, result.Manifest));
// local functions
async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName)
{
// enumerate versions in bucket
var found = await FindModuleVersionsAsync(bucketName);
if (!found.Any())
{
return(Origin : bucketName, Version : null, Manifest : null);
}
// NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null))
{
found = found.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidate => {
var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidate, moduleInfo.Origin);
var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).Result;
manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText);
// check if module is compatible with this tool
return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.CoreServicesVersion));
});
return(Origin : bucketName, Version : match, Manifest : manifest);
}
async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName)
{
// get bucket region specific S3 client
var s3Client = await GetS3ClientByBucketNameAsync(bucketName);
if (s3Client == null)
{
// nothing to do; GetS3ClientByBucketName already emitted an error
return(new List <VersionInfo>());
}
// enumerate versions in bucket
var versions = new List <VersionInfo>();
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/",
Delimiter = "/",
MaxKeys = 100,
RequestPayer = RequestPayer.Requester
};
do
{
try {
var response = await s3Client.ListObjectsV2Async(request);
versions.AddRange(response.S3Objects
.Select(s3Object => s3Object.Key.Substring(request.Prefix.Length))
.Select(found => VersionInfo.Parse(found))
.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true))
);
request.ContinuationToken = response.NextContinuationToken;
} catch (AmazonS3Exception e) when(e.Message == "Access Denied")
{
break;
}
} while(request.ContinuationToken != null);
LogInfoVerbose($"==> Found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(versions);
}
ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest)
=> new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum);
}