public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
Metadata[] modules = new Metadata[] { metadata };
foreach (ITransformer tr in _transformers)
{
modules = modules
.SelectMany(meta => tr.Transform(meta, opts))
.ToArray();
// The metadata should be valid after each step
foreach (Metadata meta in modules)
{
_validator.Validate(meta);
}
}
return(modules);
}
/// <summary>
/// Apply the locale transformation to the metadata
/// </summary>
/// <param name="metadata">Data about the module</param>
/// <returns>
/// Updated metadata with the `locales` property set
/// </returns>
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
JObject json = metadata.Json();
if (json.ContainsKey(localizationsProperty))
{
log.Debug("Localizations property already set, skipping");
// Already set, don't override (skips a bunch of file processing)
yield return(metadata);
}
else
{
CkanModule mod = CkanModule.FromJson(json.ToString());
ZipFile zip = new ZipFile(_http.DownloadPackage(
metadata.Download,
metadata.Identifier,
metadata.RemoteTimestamp
));
log.Debug("Extracting locales");
// Extract the locale names from the ZIP's cfg files
var locales = _moduleService.GetConfigFiles(mod, zip)
.Select(cfg => new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd())
.SelectMany(contents => localizationRegex.Matches(contents).Cast <Match>()
.Select(m => m.Groups["contents"].Value))
.SelectMany(contents => localeRegex.Matches(contents).Cast <Match>()
.Where(m => m.Groups["contents"].Value.Contains("="))
.Select(m => m.Groups["locale"].Value))
.Distinct()
.Memoize();
log.Debug("Locales extracted");
if (locales.Any())
{
json.SafeAdd(localizationsProperty, new JArray(locales));
log.Debug("Localizations property set");
yield return(new Metadata(json));
}
else
{
log.Debug("No localizations found");
yield return(metadata);
}
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
var json = metadata.Json();
var sortedJson = new JObject();
Log.InfoFormat("Executing property sort transformation");
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var sortedPropertyNames = json
.Properties()
.Select(i => i.Name)
.OrderBy(GetPropertySortOrder)
.ThenBy(i => i);
foreach (var propertyName in sortedPropertyNames)
{
sortedJson[propertyName] = json[propertyName];
}
var resources = json["resources"] as JObject;
if (resources != null)
{
var sortedResourcePropertyNames = resources
.Properties()
.Select(i => i.Name)
.OrderBy(GetResourcePropertySortOrder)
.ThenBy(i => i);
var sortedResources = new JObject();
foreach (var resourceProprtyName in sortedResourcePropertyNames)
{
sortedResources[resourceProprtyName] = resources[resourceProprtyName];
}
sortedJson["resources"] = sortedResources;
}
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, sortedJson);
yield return(new Metadata(sortedJson));
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref?.Source == "ksp-avc")
{
var json = metadata.Json();
Log.InfoFormat("Executing KSP-AVC $kref transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var url = new Uri(metadata.Kref.Id);
AvcVersion remoteAvc = JsonConvert.DeserializeObject <AvcVersion>(
githubSrc?.DownloadText(url)
?? httpSvc.DownloadText(CKAN.Net.GetRawUri(url))
);
json.SafeAdd("name", remoteAvc.Name);
json.Remove("$kref");
json.SafeAdd("download", remoteAvc.Download);
// Set .resources.repository based on GITHUB properties
if (remoteAvc.Github?.Username != null && remoteAvc.Github?.Repository != null)
{
// Make sure resources exist.
if (json["resources"] == null)
{
json["resources"] = new JObject();
}
var resourcesJson = (JObject)json["resources"];
resourcesJson.SafeAdd("repository", $"https://github.com/{remoteAvc.Github.Username}/{remoteAvc.Github.Repository}");
}
// Use standard KSP-AVC logic to set version and the ksp_version_* properties
AvcTransformer.ApplyVersions(json, remoteAvc);
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
yield return(new Metadata(json));
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
JObject json = metadata.Json();
var matchingKeys = kspVersionKeys
.Where(vk => json.ContainsKey(vk) &&
!string.IsNullOrEmpty((string)json[vk]) &&
(string)json[vk] != "any")
.Memoize();
if (matchingKeys.Any())
{
string msg = string.Join(", ", matchingKeys.Select(mk => $"{mk} = {json[mk]}"));
Log.DebugFormat("Enabling staging, found KSP version keys in netkan: {0}", msg);
opts.Staged = true;
opts.StagingReason = $"Game version keys found in netkan: {msg}.\r\n\r\nPlease check that their values match the forum thread.";
}
// This transformer never changes the metadata
yield return(metadata);
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
var json = metadata.Json();
JToken overrideList;
if (json.TryGetValue("x_netkan_override", out overrideList))
{
Log.InfoFormat("Executing override transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
// There's an override section, process them
Log.DebugFormat("Override section:{0}{1}", Environment.NewLine, overrideList);
if (overrideList.Type == JTokenType.Array)
{
// Sweet! We have an override. Let's walk through and see if we can find
// an applicable section for us.
foreach (var overrideStanza in overrideList)
{
ProcessOverrideStanza((JObject)overrideStanza, json);
}
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
yield return(new Metadata(json));
}
else
{
throw new Kraken(
string.Format(
"x_netkan_override expects a list of overrides, found: {0}",
overrideList));
}
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Download != null)
{
var json = metadata.Json();
string file = _http.DownloadModule(metadata);
var internalJson = _moduleService.GetInternalCkan(file);
if (internalJson != null)
{
Log.InfoFormat("Executing internal CKAN transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
foreach (var property in internalJson.Properties())
{
// We've already got the file, too late to tell us where it lives
if (property.Name == "$kref")
{
Log.DebugFormat("Skipping $kref property: {0}", property.Value);
continue;
}
json.SafeAdd(property.Name, property.Value);
}
json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, new Metadata(internalJson).SpecVersion)
.ToSpecVersionJson();
json.SafeMerge("resources", internalJson["resources"]);
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
}
yield return(new Metadata(json));
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref != null && metadata.Kref.Source == "jenkins")
{
var json = metadata.Json();
Log.InfoFormat("Executing Jenkins transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
JenkinsOptions options = json["x_netkan_jenkins"]?.ToObject <JenkinsOptions>()
?? new JenkinsOptions();
JenkinsRef jRef = new JenkinsRef(metadata.Kref);
var versions = _api.GetAllBuilds(jRef, options);
if (opts.SkipReleases.HasValue)
{
versions = versions.Skip(opts.SkipReleases.Value);
}
if (opts.Releases.HasValue)
{
versions = versions.Take(opts.Releases.Value);
}
bool returnedAny = false;
foreach (JenkinsBuild build in versions)
{
returnedAny = true;
yield return(TransformOne(metadata, metadata.Json(), build, options));
}
if (!returnedAny)
{
Log.WarnFormat("No releases found for {0}", jRef.BaseUri);
yield return(metadata);
}
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref != null && metadata.Kref.Source == "http")
{
var json = metadata.Json();
Log.InfoFormat("Executing HTTP transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
if (Uri.IsWellFormedUriString(metadata.Kref.Id, UriKind.Absolute))
{
var resolvedUri = Net.ResolveRedirect(new Uri(metadata.Kref.Id));
Log.InfoFormat("URL {0} resolved to {1}", metadata.Kref.Id, resolvedUri);
if (resolvedUri != null)
{
json.Remove("$kref");
json["download"] = resolvedUri.ToString();
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
yield return(new Metadata(json));
}
else
{
throw new Kraken("Could not resolve HTTP $kref URL, exceeded number of redirects.");
}
}
else
{
throw new Kraken("Invalid URL in HTTP $kref: " + metadata.Kref.Id);
}
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref != null && metadata.Kref.Source == "curse")
{
var json = metadata.Json();
Log.InfoFormat("Executing Curse transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
// Look up our mod on Curse by its Id.
var curseMod = _api.GetMod(metadata.Kref.Id);
var versions = curseMod.All();
if (opts.SkipReleases.HasValue)
{
versions = versions.Skip(opts.SkipReleases.Value);
}
if (opts.Releases.HasValue)
{
versions = versions.Take(opts.Releases.Value);
}
bool returnedAny = false;
foreach (CurseFile f in versions)
{
returnedAny = true;
yield return(TransformOne(metadata.Json(), curseMod, f));
}
if (!returnedAny)
{
Log.WarnFormat("No releases found for {0}", curseMod.ToString());
yield return(metadata);
}
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Download != null)
{
var json = metadata.Json();
Log.InfoFormat("Executing Download attribute transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
string file = _http.DownloadModule(metadata);
if (file != null)
{
Log.Debug("Calculating download size...");
json["download_size"] = _fileService.GetSizeBytes(file);
json["download_hash"] = new JObject();
var download_hashJson = (JObject)json["download_hash"];
Log.Debug("Calculating download SHA1...");
download_hashJson.SafeAdd("sha1", _fileService.GetFileHashSha1(file));
Log.Debug("Calculating download SHA256...");
download_hashJson.SafeAdd("sha256", _fileService.GetFileHashSha256(file));
Log.Debug("Calculating download MIME type...");
json["download_content_type"] = _fileService.GetMimetype(file);
}
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
yield return(new Metadata(json));
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
_vrefValidator.Validate(metadata);
if (metadata.Vref != null && metadata.Vref.Source == "ksp-avc")
{
var json = metadata.Json();
Log.InfoFormat("Executing internal AVC transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var noVersion = metadata.Version == null;
if (noVersion)
{
json["version"] = "0"; // TODO: DBB: Dummy version necessary to the next statement doesn't throw
}
var mod = CkanModule.FromJson(json.ToString());
if (noVersion)
{
json.Remove("version");
}
var file = _http.DownloadModule(metadata);
var avc = _moduleService.GetInternalAvc(mod, file, metadata.Vref.Id);
if (avc != null)
{
Log.Info("Found internal AVC version file");
var resourcesJson = (JObject)json["resources"];
var remoteUri = resourcesJson?["remote-avc"] != null
? new Uri((string)resourcesJson["remote-avc"])
: GetRemoteAvcUri(avc);
if (remoteUri != null)
{
if (resourcesJson == null)
{
json["resources"] = resourcesJson = new JObject();
}
resourcesJson.SafeAdd("remote-avc", remoteUri.OriginalString);
try
{
var remoteJson = _github?.DownloadText(remoteUri)
?? _http.DownloadText(remoteUri);
var remoteAvc = JsonConvert.DeserializeObject <AvcVersion>(remoteJson);
if (avc.version.CompareTo(remoteAvc.version) == 0)
{
// Local AVC and Remote AVC describe the same version, prefer
Log.Info("Remote AVC version file describes same version as local AVC version file, using it preferentially.");
avc = remoteAvc;
}
}
catch (JsonReaderException e)
{
Log.WarnFormat("Error parsing remote version file {0}: {1}",
remoteUri, e.Message);
Log.Debug(e);
}
catch (Exception e)
{
Log.WarnFormat("Error fetching remote version file {0}: {1}", remoteUri, e.Message);
Log.Debug(e);
}
}
ApplyVersions(json, avc);
// It's cool if we don't have version info at all, it's optional in the AVC spec.
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
}
yield return(new Metadata(json));
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref != null && metadata.Kref.Source == KrefSource)
{
var json = metadata.Json();
Log.InfoFormat("Executing MetaNetkan transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
// Make sure resources exist, save metanetkan
if (json["resources"] == null)
{
json["resources"] = new JObject();
}
var resourcesJson = (JObject)json["resources"];
resourcesJson.SafeAdd("metanetkan", metadata.Kref.Id);
var uri = new Uri(metadata.Kref.Id);
var targetFileText = _github?.DownloadText(uri)
?? _http.DownloadText(CKAN.Net.GetRawUri(uri));
Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText);
var targetJson = JObject.Parse(targetFileText);
var targetMetadata = new Metadata(targetJson);
if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan")
{
json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion)
.ToSpecVersionJson();
if (targetJson["$kref"] != null)
{
json["$kref"] = targetJson["$kref"];
}
else
{
json.Remove("$kref");
}
json.SafeMerge("resources", targetJson["resources"]);
foreach (var property in targetJson.Properties())
{
json.SafeAdd(property.Name, property.Value);
}
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
yield return(new Metadata(json));
}
else
{
throw new Kraken("The target of a metanetkan may not also be a metanetkan.");
}
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Vref != null && metadata.Vref.Source == "ksp-avc")
{
var json = metadata.Json();
Log.InfoFormat("Executing internal AVC transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var noVersion = metadata.Version == null;
if (noVersion)
{
json["version"] = "0"; // TODO: DBB: Dummy version necessary to the next statement doesn't throw
}
var mod = CkanModule.FromJson(json.ToString());
if (noVersion)
{
json.Remove("version");
}
var file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
var avc = _moduleService.GetInternalAvc(mod, file, metadata.Vref.Id);
if (avc != null)
{
Log.Info("Found internal AVC version file");
var remoteUri = GetRemoteAvcUri(avc);
if (remoteUri != null)
{
try
{
var remoteJson = _github?.DownloadText(remoteUri)
?? _http.DownloadText(remoteUri);
var remoteAvc = JsonConvert.DeserializeObject <AvcVersion>(remoteJson);
if (avc.version.CompareTo(remoteAvc.version) == 0)
{
// Local AVC and Remote AVC describe the same version, prefer
Log.Info("Remote AVC version file describes same version as local AVC version file, using it preferrentially.");
avc = remoteAvc;
}
}
catch (Exception e)
{
Log.InfoFormat("An error occured fetching the remote AVC version file, ignoring: {0}", e.Message);
Log.Debug(e);
}
}
ApplyVersions(json, avc);
// It's cool if we don't have version info at all, it's optional in the AVC spec.
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
}
yield return(new Metadata(json));
}
else
{
yield return(metadata);
}
}
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts)
{
if (metadata.Kref != null && metadata.Kref.Source == "github")
{
var json = metadata.Json();
Log.InfoFormat("Executing GitHub transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var useSourceAchive = false;
var githubMetadata = (JObject)json["x_netkan_github"];
if (githubMetadata != null)
{
var githubUseSourceArchive = (bool?)githubMetadata["use_source_archive"];
if (githubUseSourceArchive != null)
{
useSourceAchive = githubUseSourceArchive.Value;
}
}
var ghRef = new GithubRef(metadata.Kref, useSourceAchive, _matchPreleases);
// Get the GitHub repository
var ghRepo = _api.GetRepo(ghRef);
if (ghRepo.Archived)
{
Log.Warn("Repo is archived, consider freezing");
}
var releases = _api.GetAllReleases(ghRef);
if (opts.SkipReleases.HasValue)
{
releases = releases.Skip(opts.SkipReleases.Value);
}
if (opts.Releases.HasValue)
{
releases = releases.Take(opts.Releases.Value);
}
bool returnedAny = false;
foreach (GithubRelease rel in releases)
{
if (ghRef.VersionFromAsset != null)
{
Log.DebugFormat("Found version_from_asset regex, inflating all assets");
foreach (var asset in rel.Assets)
{
var match = ghRef.VersionFromAsset.Match(asset.Name);
if (!match.Success)
{
continue;
}
var extractedVersion = match.Groups["version"];
if (!extractedVersion.Success)
{
throw new Exception("version_from_asset contains no 'version' capturing group");
}
returnedAny = true;
yield return(TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, asset, extractedVersion.Value));
}
}
else
{
if (rel.Assets.Count > 1)
{
Log.WarnFormat("Multiple assets found for {0} {1} without `version_from_asset`",
metadata.Identifier, rel.Tag);
}
returnedAny = true;
yield return(TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, rel.Assets.FirstOrDefault(), rel.Tag.ToString()));
}
}
if (!returnedAny)
{
if (ghRef.Filter != Constants.DefaultAssetMatchPattern)
{
Log.WarnFormat("No releases found for {0} with asset_match {1}", ghRef.Repository, ghRef.Filter);
}
else if (ghRef.VersionFromAsset != null)
{
Log.WarnFormat("No releases found for {0} with version_from_asset {1}", ghRef.Repository, ghRef.VersionFromAsset);
}
else
{
Log.WarnFormat("No releases found for {0}", ghRef.Repository);
}
yield return(metadata);
}
}
else
{
yield return(metadata);
}
}