public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { Metadata[] modules = new Metadata[] { metadata }; foreach (ITransformer tr in _transformers) { modules = modules .SelectMany(meta => tr.Transform(meta, opts)) .ToArray(); // The metadata should be valid after each step foreach (Metadata meta in modules) { _validator.Validate(meta); } } return(modules); }
/// <summary> /// Apply the locale transformation to the metadata /// </summary> /// <param name="metadata">Data about the module</param> /// <returns> /// Updated metadata with the `locales` property set /// </returns> public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { JObject json = metadata.Json(); if (json.ContainsKey(localizationsProperty)) { log.Debug("Localizations property already set, skipping"); // Already set, don't override (skips a bunch of file processing) yield return(metadata); } else { CkanModule mod = CkanModule.FromJson(json.ToString()); ZipFile zip = new ZipFile(_http.DownloadPackage( metadata.Download, metadata.Identifier, metadata.RemoteTimestamp )); log.Debug("Extracting locales"); // Extract the locale names from the ZIP's cfg files var locales = _moduleService.GetConfigFiles(mod, zip) .Select(cfg => new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd()) .SelectMany(contents => localizationRegex.Matches(contents).Cast <Match>() .Select(m => m.Groups["contents"].Value)) .SelectMany(contents => localeRegex.Matches(contents).Cast <Match>() .Where(m => m.Groups["contents"].Value.Contains("=")) .Select(m => m.Groups["locale"].Value)) .Distinct() .Memoize(); log.Debug("Locales extracted"); if (locales.Any()) { json.SafeAdd(localizationsProperty, new JArray(locales)); log.Debug("Localizations property set"); yield return(new Metadata(json)); } else { log.Debug("No localizations found"); yield return(metadata); } } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { var json = metadata.Json(); var sortedJson = new JObject(); Log.InfoFormat("Executing property sort transformation"); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var sortedPropertyNames = json .Properties() .Select(i => i.Name) .OrderBy(GetPropertySortOrder) .ThenBy(i => i); foreach (var propertyName in sortedPropertyNames) { sortedJson[propertyName] = json[propertyName]; } var resources = json["resources"] as JObject; if (resources != null) { var sortedResourcePropertyNames = resources .Properties() .Select(i => i.Name) .OrderBy(GetResourcePropertySortOrder) .ThenBy(i => i); var sortedResources = new JObject(); foreach (var resourceProprtyName in sortedResourcePropertyNames) { sortedResources[resourceProprtyName] = resources[resourceProprtyName]; } sortedJson["resources"] = sortedResources; } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, sortedJson); yield return(new Metadata(sortedJson)); }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref?.Source == "ksp-avc") { var json = metadata.Json(); Log.InfoFormat("Executing KSP-AVC $kref transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var url = new Uri(metadata.Kref.Id); AvcVersion remoteAvc = JsonConvert.DeserializeObject <AvcVersion>( githubSrc?.DownloadText(url) ?? httpSvc.DownloadText(CKAN.Net.GetRawUri(url)) ); json.SafeAdd("name", remoteAvc.Name); json.Remove("$kref"); json.SafeAdd("download", remoteAvc.Download); // Set .resources.repository based on GITHUB properties if (remoteAvc.Github?.Username != null && remoteAvc.Github?.Repository != null) { // Make sure resources exist. if (json["resources"] == null) { json["resources"] = new JObject(); } var resourcesJson = (JObject)json["resources"]; resourcesJson.SafeAdd("repository", $"https://github.com/{remoteAvc.Github.Username}/{remoteAvc.Github.Repository}"); } // Use standard KSP-AVC logic to set version and the ksp_version_* properties AvcTransformer.ApplyVersions(json, remoteAvc); Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); yield return(new Metadata(json)); } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { JObject json = metadata.Json(); var matchingKeys = kspVersionKeys .Where(vk => json.ContainsKey(vk) && !string.IsNullOrEmpty((string)json[vk]) && (string)json[vk] != "any") .Memoize(); if (matchingKeys.Any()) { string msg = string.Join(", ", matchingKeys.Select(mk => $"{mk} = {json[mk]}")); Log.DebugFormat("Enabling staging, found KSP version keys in netkan: {0}", msg); opts.Staged = true; opts.StagingReason = $"Game version keys found in netkan: {msg}.\r\n\r\nPlease check that their values match the forum thread."; } // This transformer never changes the metadata yield return(metadata); }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { var json = metadata.Json(); JToken overrideList; if (json.TryGetValue("x_netkan_override", out overrideList)) { Log.InfoFormat("Executing override transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); // There's an override section, process them Log.DebugFormat("Override section:{0}{1}", Environment.NewLine, overrideList); if (overrideList.Type == JTokenType.Array) { // Sweet! We have an override. Let's walk through and see if we can find // an applicable section for us. foreach (var overrideStanza in overrideList) { ProcessOverrideStanza((JObject)overrideStanza, json); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); yield return(new Metadata(json)); } else { throw new Kraken( string.Format( "x_netkan_override expects a list of overrides, found: {0}", overrideList)); } } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Download != null) { var json = metadata.Json(); string file = _http.DownloadModule(metadata); var internalJson = _moduleService.GetInternalCkan(file); if (internalJson != null) { Log.InfoFormat("Executing internal CKAN transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); foreach (var property in internalJson.Properties()) { // We've already got the file, too late to tell us where it lives if (property.Name == "$kref") { Log.DebugFormat("Skipping $kref property: {0}", property.Value); continue; } json.SafeAdd(property.Name, property.Value); } json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, new Metadata(internalJson).SpecVersion) .ToSpecVersionJson(); json.SafeMerge("resources", internalJson["resources"]); Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); } yield return(new Metadata(json)); } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref != null && metadata.Kref.Source == "jenkins") { var json = metadata.Json(); Log.InfoFormat("Executing Jenkins transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); JenkinsOptions options = json["x_netkan_jenkins"]?.ToObject <JenkinsOptions>() ?? new JenkinsOptions(); JenkinsRef jRef = new JenkinsRef(metadata.Kref); var versions = _api.GetAllBuilds(jRef, options); if (opts.SkipReleases.HasValue) { versions = versions.Skip(opts.SkipReleases.Value); } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); } bool returnedAny = false; foreach (JenkinsBuild build in versions) { returnedAny = true; yield return(TransformOne(metadata, metadata.Json(), build, options)); } if (!returnedAny) { Log.WarnFormat("No releases found for {0}", jRef.BaseUri); yield return(metadata); } } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref != null && metadata.Kref.Source == "http") { var json = metadata.Json(); Log.InfoFormat("Executing HTTP transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); if (Uri.IsWellFormedUriString(metadata.Kref.Id, UriKind.Absolute)) { var resolvedUri = Net.ResolveRedirect(new Uri(metadata.Kref.Id)); Log.InfoFormat("URL {0} resolved to {1}", metadata.Kref.Id, resolvedUri); if (resolvedUri != null) { json.Remove("$kref"); json["download"] = resolvedUri.ToString(); Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); yield return(new Metadata(json)); } else { throw new Kraken("Could not resolve HTTP $kref URL, exceeded number of redirects."); } } else { throw new Kraken("Invalid URL in HTTP $kref: " + metadata.Kref.Id); } } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref != null && metadata.Kref.Source == "curse") { var json = metadata.Json(); Log.InfoFormat("Executing Curse transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); // Look up our mod on Curse by its Id. var curseMod = _api.GetMod(metadata.Kref.Id); var versions = curseMod.All(); if (opts.SkipReleases.HasValue) { versions = versions.Skip(opts.SkipReleases.Value); } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); } bool returnedAny = false; foreach (CurseFile f in versions) { returnedAny = true; yield return(TransformOne(metadata.Json(), curseMod, f)); } if (!returnedAny) { Log.WarnFormat("No releases found for {0}", curseMod.ToString()); yield return(metadata); } } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Download != null) { var json = metadata.Json(); Log.InfoFormat("Executing Download attribute transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); string file = _http.DownloadModule(metadata); if (file != null) { Log.Debug("Calculating download size..."); json["download_size"] = _fileService.GetSizeBytes(file); json["download_hash"] = new JObject(); var download_hashJson = (JObject)json["download_hash"]; Log.Debug("Calculating download SHA1..."); download_hashJson.SafeAdd("sha1", _fileService.GetFileHashSha1(file)); Log.Debug("Calculating download SHA256..."); download_hashJson.SafeAdd("sha256", _fileService.GetFileHashSha256(file)); Log.Debug("Calculating download MIME type..."); json["download_content_type"] = _fileService.GetMimetype(file); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); yield return(new Metadata(json)); } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { _vrefValidator.Validate(metadata); if (metadata.Vref != null && metadata.Vref.Source == "ksp-avc") { var json = metadata.Json(); Log.InfoFormat("Executing internal AVC transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var noVersion = metadata.Version == null; if (noVersion) { json["version"] = "0"; // TODO: DBB: Dummy version necessary to the next statement doesn't throw } var mod = CkanModule.FromJson(json.ToString()); if (noVersion) { json.Remove("version"); } var file = _http.DownloadModule(metadata); var avc = _moduleService.GetInternalAvc(mod, file, metadata.Vref.Id); if (avc != null) { Log.Info("Found internal AVC version file"); var resourcesJson = (JObject)json["resources"]; var remoteUri = resourcesJson?["remote-avc"] != null ? new Uri((string)resourcesJson["remote-avc"]) : GetRemoteAvcUri(avc); if (remoteUri != null) { if (resourcesJson == null) { json["resources"] = resourcesJson = new JObject(); } resourcesJson.SafeAdd("remote-avc", remoteUri.OriginalString); try { var remoteJson = _github?.DownloadText(remoteUri) ?? _http.DownloadText(remoteUri); var remoteAvc = JsonConvert.DeserializeObject <AvcVersion>(remoteJson); if (avc.version.CompareTo(remoteAvc.version) == 0) { // Local AVC and Remote AVC describe the same version, prefer Log.Info("Remote AVC version file describes same version as local AVC version file, using it preferentially."); avc = remoteAvc; } } catch (JsonReaderException e) { Log.WarnFormat("Error parsing remote version file {0}: {1}", remoteUri, e.Message); Log.Debug(e); } catch (Exception e) { Log.WarnFormat("Error fetching remote version file {0}: {1}", remoteUri, e.Message); Log.Debug(e); } } ApplyVersions(json, avc); // It's cool if we don't have version info at all, it's optional in the AVC spec. Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); } yield return(new Metadata(json)); } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref != null && metadata.Kref.Source == KrefSource) { var json = metadata.Json(); Log.InfoFormat("Executing MetaNetkan transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); // Make sure resources exist, save metanetkan if (json["resources"] == null) { json["resources"] = new JObject(); } var resourcesJson = (JObject)json["resources"]; resourcesJson.SafeAdd("metanetkan", metadata.Kref.Id); var uri = new Uri(metadata.Kref.Id); var targetFileText = _github?.DownloadText(uri) ?? _http.DownloadText(CKAN.Net.GetRawUri(uri)); Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText); var targetJson = JObject.Parse(targetFileText); var targetMetadata = new Metadata(targetJson); if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan") { json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion) .ToSpecVersionJson(); if (targetJson["$kref"] != null) { json["$kref"] = targetJson["$kref"]; } else { json.Remove("$kref"); } json.SafeMerge("resources", targetJson["resources"]); foreach (var property in targetJson.Properties()) { json.SafeAdd(property.Name, property.Value); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); yield return(new Metadata(json)); } else { throw new Kraken("The target of a metanetkan may not also be a metanetkan."); } } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Vref != null && metadata.Vref.Source == "ksp-avc") { var json = metadata.Json(); Log.InfoFormat("Executing internal AVC transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var noVersion = metadata.Version == null; if (noVersion) { json["version"] = "0"; // TODO: DBB: Dummy version necessary to the next statement doesn't throw } var mod = CkanModule.FromJson(json.ToString()); if (noVersion) { json.Remove("version"); } var file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp); var avc = _moduleService.GetInternalAvc(mod, file, metadata.Vref.Id); if (avc != null) { Log.Info("Found internal AVC version file"); var remoteUri = GetRemoteAvcUri(avc); if (remoteUri != null) { try { var remoteJson = _github?.DownloadText(remoteUri) ?? _http.DownloadText(remoteUri); var remoteAvc = JsonConvert.DeserializeObject <AvcVersion>(remoteJson); if (avc.version.CompareTo(remoteAvc.version) == 0) { // Local AVC and Remote AVC describe the same version, prefer Log.Info("Remote AVC version file describes same version as local AVC version file, using it preferrentially."); avc = remoteAvc; } } catch (Exception e) { Log.InfoFormat("An error occured fetching the remote AVC version file, ignoring: {0}", e.Message); Log.Debug(e); } } ApplyVersions(json, avc); // It's cool if we don't have version info at all, it's optional in the AVC spec. Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); } yield return(new Metadata(json)); } else { yield return(metadata); } }
public IEnumerable <Metadata> Transform(Metadata metadata, TransformOptions opts) { if (metadata.Kref != null && metadata.Kref.Source == "github") { var json = metadata.Json(); Log.InfoFormat("Executing GitHub transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var useSourceAchive = false; var githubMetadata = (JObject)json["x_netkan_github"]; if (githubMetadata != null) { var githubUseSourceArchive = (bool?)githubMetadata["use_source_archive"]; if (githubUseSourceArchive != null) { useSourceAchive = githubUseSourceArchive.Value; } } var ghRef = new GithubRef(metadata.Kref, useSourceAchive, _matchPreleases); // Get the GitHub repository var ghRepo = _api.GetRepo(ghRef); if (ghRepo.Archived) { Log.Warn("Repo is archived, consider freezing"); } var releases = _api.GetAllReleases(ghRef); if (opts.SkipReleases.HasValue) { releases = releases.Skip(opts.SkipReleases.Value); } if (opts.Releases.HasValue) { releases = releases.Take(opts.Releases.Value); } bool returnedAny = false; foreach (GithubRelease rel in releases) { if (ghRef.VersionFromAsset != null) { Log.DebugFormat("Found version_from_asset regex, inflating all assets"); foreach (var asset in rel.Assets) { var match = ghRef.VersionFromAsset.Match(asset.Name); if (!match.Success) { continue; } var extractedVersion = match.Groups["version"]; if (!extractedVersion.Success) { throw new Exception("version_from_asset contains no 'version' capturing group"); } returnedAny = true; yield return(TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, asset, extractedVersion.Value)); } } else { if (rel.Assets.Count > 1) { Log.WarnFormat("Multiple assets found for {0} {1} without `version_from_asset`", metadata.Identifier, rel.Tag); } returnedAny = true; yield return(TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, rel.Assets.FirstOrDefault(), rel.Tag.ToString())); } } if (!returnedAny) { if (ghRef.Filter != Constants.DefaultAssetMatchPattern) { Log.WarnFormat("No releases found for {0} with asset_match {1}", ghRef.Repository, ghRef.Filter); } else if (ghRef.VersionFromAsset != null) { Log.WarnFormat("No releases found for {0} with version_from_asset {1}", ghRef.Repository, ghRef.VersionFromAsset); } else { Log.WarnFormat("No releases found for {0}", ghRef.Repository); } yield return(metadata); } } else { yield return(metadata); } }