public void TestFixtureSetup()
{
var tempDirectory = Path.Combine(Path.GetTempPath(), "CKAN", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(tempDirectory);
_cache = new NetFileCache(tempDirectory);
}
public void RemoveCache()
{
cache.Dispose();
cache = null;
module_cache.Dispose();
module_cache = null;
Directory.Delete(cache_dir, true);
}
public void TestFixtureSetup()
{
_cachePath = Path.Combine(Path.GetTempPath(), "CKAN");
var path = Path.Combine(_cachePath, Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(path);
_cache = new NetFileCache(path);
}
public void ZipValid_ContainsFilenameWithUnicodeChars_Valid()
{
bool valid = false;
string reason = null;
Assert.DoesNotThrow(() =>
valid = NetFileCache.ZipValid(TestData.ZipWithUnicodeChars, out reason));
Assert.IsTrue(valid, reason);
}
public string DownloadPackage(Uri url, string identifier, DateTime?updated)
{
_requestedURLs.Add(url);
var cachedFile = _cache.GetCachedFilename(url, updated);
if (!string.IsNullOrWhiteSpace(cachedFile))
{
return(cachedFile);
}
else
{
var downloadedFile = Net.Download(url);
string extension;
switch (FileIdentifier.IdentifyFile(downloadedFile))
{
case FileType.ASCII:
extension = "txt";
break;
case FileType.GZip:
extension = "gz";
break;
case FileType.Tar:
extension = "tar";
break;
case FileType.TarGz:
extension = "tar.gz";
break;
case FileType.Zip:
extension = "zip";
string invalidReason;
if (!NetFileCache.ZipValid(downloadedFile, out invalidReason))
{
throw new Kraken($"{downloadedFile} is not a valid ZIP file: {invalidReason}");
}
break;
default:
extension = "ckan-package";
break;
}
return(_cache.Store(
url,
downloadedFile,
string.Format("netkan-{0}.{1}", identifier, extension),
move: true
));
}
}
public ModDirectoryManager(string repoUrlPrefix, string repoLocalPath, string subDirectoryName, IModFileNormalizer modNormalizer, NetFileCache netFileCache)
{
this.repoUrlPrefix = repoUrlPrefix;
this.repoLocalPath = repoLocalPath;
this.subDirectoryName = subDirectoryName;
this.modNormalizer = modNormalizer;
this.cache = netFileCache;
Directory.CreateDirectory(RepoModsDirectoryPath);
Directory.CreateDirectory(RepoPacksDirectoryPath);
}
private static void PurgeDownloads(IHttpService http, NetFileCache cache)
{
if (http != null && cache != null)
{
foreach (Uri url in http.RequestedURLs)
{
cache.Remove(url);
}
}
}
public string Download(string identifier, NetFileCache cache)
{
log.DebugFormat("Downloading {0}", download);
string filename = ModuleInstaller.CachedOrDownload(identifier, version, download, cache);
log.Debug("Downloaded.");
return filename;
}
public string Download(string identifier, NetFileCache cache)
{
log.DebugFormat("Downloading {0}", download_path);
string filename = ModuleInstaller.CachedOrDownload(identifier, friendly_version, new Uri(download_path), cache);
log.Debug("Downloaded.");
return(filename);
}
public string Download(string identifier, NetFileCache cache)
{
log.DebugFormat("Downloading {0}", download);
string filename = ModuleInstaller.CachedOrDownload(identifier, version, download, cache);
log.Debug("Downloaded.");
return(filename);
}
internal static JObject HTTP(JObject orig_metadata, string remote_id, NetFileCache cache, IUser user)
{
var metadata = orig_metadata;
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/http")
{
log.InfoFormat("Inflating from HTTP download... {0}", metadata[expand_token]);
metadata["download"] = remote_id;
metadata["version"] = "0.0.0"; // add a dummy version that will be replaced by the KSP-AVC parser later
metadata.Remove(expand_token);
var remote_uri = new Uri(remote_id);
var downloaded_file = Net.Download(remote_uri);
var module = CkanModule.FromJson(metadata.ToString());
if (metadata[version_token] != null && (metadata[version_token].ToString()).StartsWith("#/ckan/ksp-avc"))
{
// TODO pass the correct vref here...
var versionRemote = FindVersionRemote(metadata, metadata[version_token].ToString());
metadata.Remove(version_token);
try
{
AVC avc = AVC.FromZipFile(module, downloaded_file, versionRemote.id);
avc.InflateMetadata(metadata, null, null);
metadata["version"] = avc.version.ToString();
module.version = avc.version;
}
catch (JsonReaderException)
{
user.RaiseMessage("Bad embedded KSP-AVC file for {0}, halting.", module);
return(null);
}
// If we've done this, we need to re-inflate our mod, too.
module = CkanModule.FromJson(metadata.ToString());
}
else
{
throw new Kraken("No $vref specified, $kref HTTP method requires it, bailing out..");
}
ModuleInstaller.CachedOrDownload(module.identifier, module.version, module.download, cache);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
// metadata["download"] = metadata["download"].ToString() + '#' + metadata["version"].ToString();
return(metadata);
}
private static void PurgeDownloads(IHttpService http, NetFileCache cache)
{
log.Debug("Deleting downloads for failed inflation");
if (http != null && cache != null)
{
foreach (Uri url in http.RequestedURLs)
{
cache.Remove(url);
}
}
}
public CachingHttpService(NetFileCache cache)
{
_cache = cache;
CurlSharp.Curl.GlobalInit(CurlInitFlag.All);
_curl = new CurlEasy { UserAgent = Net.UserAgentString };
var caBundle = ResolveCurlCaBundle();
if (caBundle != null)
{
_curl.CaInfo = caBundle;
}
}
public void ZipValid_ContainsFilenameWithBadChars_NoException()
{
bool valid = false;
string reason = "";
Assert.DoesNotThrow(() =>
valid = NetFileCache.ZipValid(TestData.ZipWithBadChars, out reason));
// The file is considered valid on Linux;
// only check the reason if found invalid
if (!valid)
{
Assert.AreEqual(reason, "Illegal characters in path.");
}
}
/// <summary>
/// Fetch Metadata from (successful) Jenkins builds
/// Returns a JObject that should be a fully formed CKAN file.
/// </summary>
internal static JObject Jenkins(JObject orig_metadata, string remote_id, NetFileCache cache)
{
string versionBase = (string)orig_metadata ["x_ci_version_base"];
log.DebugFormat("versionBase: {0}", versionBase);
JObject resources = (JObject)orig_metadata ["resources"];
log.DebugFormat("resources: {0}", resources);
string baseUri = (string)resources ["ci"];
if (baseUri == null)
{
// Fallback, we don't have the defined resource 'ci' in the schema yet...
baseUri = (string)resources ["x_ci"];
}
log.DebugFormat("baseUri: {0}", baseUri);
JenkinsBuild build = JenkinsAPI.GetLatestBuild(baseUri, versionBase, true);
Version version = build.version;
log.DebugFormat("Mod: {0} {1}", remote_id, version);
// Find the latest download.
string filename = build.Download((string)orig_metadata["identifier"], cache);
JObject metadata = MetadataFromFileOrDefault(filename, orig_metadata);
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/kerbalstuff")
{
log.InfoFormat("Inflating from Jenkins... {0}", metadata[expand_token]);
build.InflateMetadata(metadata, filename, null);
metadata.Remove(expand_token);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
return(metadata);
}
public CachingHttpService(NetFileCache cache)
{
_cache = cache;
CurlSharp.Curl.GlobalInit(CurlInitFlag.All);
_curl = new CurlEasy {
UserAgent = Net.UserAgentString
};
var caBundle = ResolveCurlCaBundle();
if (caBundle != null)
{
_curl.CaInfo = caBundle;
}
}
public void Setup()
{
// Make sure curl is all set up.
Curl.Init();
// Give us a registry to play with.
ksp = new DisposableKSP();
manager = CKAN.RegistryManager.Instance(ksp.KSP);
registry = manager.registry;
registry.ClearDlls();
registry.Installed().Clear();
// Make sure we have a registry we can use.
CKAN.Repo.Update(manager, ksp.KSP, new NullUser(), TestData.TestKANZip());
// Ready our downloader.
async = new CKAN.NetAsyncModulesDownloader(new NullUser());
// General shortcuts
cache = ksp.KSP.Cache;
}
public void Setup()
{
// Make sure curl is all set up.
Curl.Init();
// Give us a registry to play with.
ksp = new DisposableKSP();
registry = ksp.KSP.Registry;
registry.ClearAvailable();
registry.ClearPreexistingModules();
registry.Installed().Clear();
// Make sure we have a registry we can use.
CKAN.Repo.UpdateRegistry(TestData.TestKANZip(), registry, ksp.KSP, new NullUser());
// Ready our downloader.
async = new CKAN.NetAsyncModulesDownloader(new NullUser(), ksp.KSP.tryGetFactorioAuthData());
// General shortcuts
cache = ksp.KSP.Cache;
}
public void ZipValid_ContainsFilenameWithBadChars_NoException()
{
// We want to inspect a localized error message below.
// Switch to English to ensure it's what we expect.
CultureInfo origUICulture = Thread.CurrentThread.CurrentUICulture;
Thread.CurrentThread.CurrentUICulture = CultureInfo.InvariantCulture;
bool valid = false;
string reason = "";
Assert.DoesNotThrow(() =>
valid = NetFileCache.ZipValid(TestData.ZipWithBadChars, out reason));
// The file is considered valid on Linux;
// only check the reason if found invalid
if (!valid)
{
Assert.AreEqual(reason, "Illegal characters in path.");
}
// Switch back to the original locale
Thread.CurrentThread.CurrentUICulture = origUICulture;
}
public static int Main(string[] args)
{
// Keep these for purging downloads in the exception handler
NetFileCache cache = null;
IHttpService http = null;
try
{
ProcessArgs(args);
// Force-allow TLS 1.2 for HTTPS URLs, because GitHub requires it.
// This is on by default in .NET 4.6, but not in 4.5.
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// If we see the --version flag, then display our build info
// and exit.
if (Options.Version)
{
Console.WriteLine(Meta.GetVersion(VersionFormat.Full));
return(ExitOk);
}
if (Options.File != null)
{
Log.InfoFormat("Transforming {0}", Options.File);
var moduleService = new ModuleService();
var fileService = new FileService();
cache = FindCache(
new KSPManager(new ConsoleUser(false)),
new Win32Registry()
);
http = new CachingHttpService(cache, Options.OverwriteCache);
var netkan = ReadNetkan();
Log.Info("Finished reading input");
new NetkanValidator().Validate(netkan);
Log.Info("Input successfully passed pre-validation");
var transformer = new NetkanTransformer(
http,
fileService,
moduleService,
Options.GitHubToken,
Options.PreRelease,
ParseReleases(Options.Releases)
);
IEnumerable <Metadata> ckans = transformer.Transform(netkan);
Log.Info("Finished transformation");
foreach (Metadata ckan in ckans)
{
new CkanValidator(netkan, http, moduleService).Validate(ckan);
Log.Info("Output successfully passed post-validation");
WriteCkan(ckan);
}
}
else
{
Log.Fatal(
"Usage: netkan [--verbose|--debug] [--debugger] [--prerelease] [--outputdir=...] <filename>"
);
return(ExitBadOpt);
}
}
catch (Exception e)
{
e = e.GetBaseException() ?? e;
// Purge anything we download for a failed indexing attempt from the cache to allow re-downloads
PurgeDownloads(http, cache);
Log.Fatal(e.Message);
if (Options == null || Options.Debug)
{
Log.Fatal(e.StackTrace);
}
return(ExitError);
}
return(ExitOk);
}
/// <summary>
/// Shows information about the mod.
/// </summary>
/// <returns>Success status.</returns>
/// <param name="module">The module to show.</param>
public int ShowMod(CkanModule module)
{
#region Abstract and description
if (!string.IsNullOrEmpty(module.@abstract))
{
user.RaiseMessage("{0}: {1}", module.name, module.@abstract);
}
else
{
user.RaiseMessage("{0}", module.name);
}
if (!string.IsNullOrEmpty(module.description))
{
user.RaiseMessage("");
user.RaiseMessage("{0}", module.description);
}
#endregion
#region General info (author, version...)
user.RaiseMessage("");
user.RaiseMessage("Module info:");
user.RaiseMessage(" Version:\t{0}", module.version);
if (module.author != null)
{
user.RaiseMessage(" Authors:\t{0}", string.Join(", ", module.author));
}
else
{
// Did you know that authors are optional in the spec?
// You do now. #673.
user.RaiseMessage(" Authors:\tUNKNOWN");
}
if (module.release_status != null)
{
user.RaiseMessage(" Status:\t{0}", module.release_status);
}
user.RaiseMessage(" License:\t{0}", string.Join(", ", module.license));
if (module.Tags != null && module.Tags.Count > 0)
{
// Need an extra space before the tab to line up with other fields
user.RaiseMessage(" Tags: \t{0}", string.Join(", ", module.Tags));
}
if (module.localizations != null && module.localizations.Length > 0)
{
user.RaiseMessage(" Languages:\t{0}", string.Join(", ", module.localizations.OrderBy(l => l)));
}
#endregion
#region Relationships
if (module.depends != null && module.depends.Count > 0)
{
user.RaiseMessage("");
user.RaiseMessage("Depends:");
foreach (RelationshipDescriptor dep in module.depends)
{
user.RaiseMessage(" - {0}", RelationshipToPrintableString(dep));
}
}
if (module.recommends != null && module.recommends.Count > 0)
{
user.RaiseMessage("");
user.RaiseMessage("Recommends:");
foreach (RelationshipDescriptor dep in module.recommends)
{
user.RaiseMessage(" - {0}", RelationshipToPrintableString(dep));
}
}
if (module.suggests != null && module.suggests.Count > 0)
{
user.RaiseMessage("");
user.RaiseMessage("Suggests:");
foreach (RelationshipDescriptor dep in module.suggests)
{
user.RaiseMessage(" - {0}", RelationshipToPrintableString(dep));
}
}
if (module.provides != null && module.provides.Count > 0)
{
user.RaiseMessage("");
user.RaiseMessage("Provides:");
foreach (string prov in module.provides)
{
user.RaiseMessage(" - {0}", prov);
}
}
#endregion
if (module.resources != null)
{
user.RaiseMessage("");
user.RaiseMessage("Resources:");
if (module.resources.homepage != null)
{
user.RaiseMessage(" Home page:\t{0}", Uri.EscapeUriString(module.resources.homepage.ToString()));
}
if (module.resources.manual != null)
{
user.RaiseMessage(" Manual:\t{0}", Uri.EscapeUriString(module.resources.manual.ToString()));
}
if (module.resources.spacedock != null)
{
user.RaiseMessage(" SpaceDock:\t{0}", Uri.EscapeUriString(module.resources.spacedock.ToString()));
}
if (module.resources.repository != null)
{
user.RaiseMessage(" Repository:\t{0}", Uri.EscapeUriString(module.resources.repository.ToString()));
}
if (module.resources.bugtracker != null)
{
user.RaiseMessage(" Bug tracker:\t{0}", Uri.EscapeUriString(module.resources.bugtracker.ToString()));
}
if (module.resources.curse != null)
{
user.RaiseMessage(" Curse:\t{0}", Uri.EscapeUriString(module.resources.curse.ToString()));
}
if (module.resources.store != null)
{
user.RaiseMessage(" Store:\t{0}", Uri.EscapeUriString(module.resources.store.ToString()));
}
if (module.resources.steamstore != null)
{
user.RaiseMessage(" Steam store:\t{0}", Uri.EscapeUriString(module.resources.steamstore.ToString()));
}
if (module.resources.remoteAvc != null)
{
user.RaiseMessage(" Version file:\t{0}", Uri.EscapeUriString(module.resources.remoteAvc.ToString()));
}
}
if (!module.IsDLC)
{
// Compute the CKAN filename.
string file_uri_hash = NetFileCache.CreateURLHash(module.download);
string file_name = CkanModule.StandardName(module.identifier, module.version);
user.RaiseMessage("");
user.RaiseMessage("Filename: {0}", file_uri_hash + "-" + file_name);
}
return(Exit.OK);
}
private const string version_token = "$vref"; // It'd be nice if we could have repeated $krefs.
public static int Main(string[] args)
{
BasicConfigurator.Configure();
LogManager.GetRepository().Threshold = Level.Error;
var user = new ConsoleUser(false);
var options = new CmdLineOptions();
Parser.Default.ParseArgumentsStrict(args, options);
if (options.Verbose)
{
LogManager.GetRepository().Threshold = Level.Info;
}
else if (options.Debug)
{
LogManager.GetRepository().Threshold = Level.Debug;
}
if (options.NetUserAgent != null)
{
Net.UserAgentString = options.NetUserAgent;
}
if (options.File == null)
{
log.Fatal("Usage: netkan [--verbose|--debug] [--prerelease] [--outputdir=...] <filename>");
return(EXIT_BADOPT);
}
NetFileCache cache = FindCache(options, new KSPManager(user), user);
log.InfoFormat("Processing {0}", options.File);
JObject json = JsonFromFile(options.File);
NetKanRemote remote = FindRemote(json);
JObject metadata;
switch (remote.source)
{
case "kerbalstuff":
metadata = KerbalStuff(json, remote.id, cache);
break;
case "jenkins":
metadata = Jenkins(json, remote.id, cache);
break;
case "github":
if (options.GitHubToken != null)
{
GithubAPI.SetCredentials(options.GitHubToken);
}
metadata = GitHub(json, remote.id, options.PreRelease, cache);
break;
case "http":
metadata = HTTP(json, remote.id, cache, user);
break;
default:
log.FatalFormat("Unknown remote source: {0}", remote.source);
return(EXIT_ERROR);
}
if (metadata == null)
{
log.Error("There was an error, aborting");
return(EXIT_ERROR);
}
// Make sure that at the very least this validates against our own
// internal model.
CkanModule mod = CkanModule.FromJson(metadata.ToString());
// Make sure our identifiers match.
if (mod.identifier != (string)json["identifier"])
{
log.FatalFormat("Error: Have metadata for {0}, but wanted {1}", mod.identifier, json["identifier"]);
return(EXIT_ERROR);
}
// Find our cached file, we'll need it later.
string file = cache.GetCachedZip(mod.download);
if (file == null)
{
log.FatalFormat("Error: Unable to find {0} in the cache", mod.identifier);
return(EXIT_ERROR);
}
// Make sure this would actually generate an install
try
{
ModuleInstaller.FindInstallableFiles(mod, file, null);
}
catch (BadMetadataKraken kraken)
{
log.FatalFormat("Error: Bad metadata for {0}: {1}", kraken.module, kraken.Message);
return(EXIT_ERROR);
}
// Now inflate our original metadata from AVC, if we have it.
// The reason we inflate our metadata and not the module is that in the module we can't
// tell if there's been an override of the version fields or not.
foreach (string vref in vrefs(metadata))
{
log.DebugFormat("Expanding vref {0}", vref);
if (vref.StartsWith("#/ckan/ksp-avc"))
{
// HTTP has already included the KSP-AVC data as it needs it earlier in the process
if (remote.source != "http")
{
var versionRemote = FindVersionRemote(metadata, vref);
try
{
AVC avc = AVC.FromZipFile(mod, file, versionRemote.id);
// TODO check why the AVC file can be null...
if (avc != null)
{
avc.InflateMetadata(metadata, null, null);
}
}
catch (JsonReaderException)
{
user.RaiseMessage("Bad embedded KSP-AVC file for {0}, halting.", mod);
return(EXIT_ERROR);
}
}
}
if (vref.StartsWith("#/ckan/kerbalstuff"))
{
log.DebugFormat("Kerbalstuff vref: {0}", vref);
Match match = Regex.Match(vref, @"^#/ckan/([^/]+)/(.+)");
if (!match.Success)
{
// TODO: Have a proper kraken class!
user.RaiseMessage("Cannot find remote and ID in vref: {0}, halting.", vref);
return(EXIT_ERROR);
}
string remote_id = match.Groups [2].ToString();
log.DebugFormat("Kerbalstuff id : {0}", remote_id);
try
{
KSMod ks = KSAPI.Mod(Convert.ToInt32(remote_id));
if (ks != null)
{
KSVersion version = new KSVersion();
version.friendly_version = mod.version;
ks.InflateMetadata(metadata, file, version);
}
}
catch (JsonReaderException)
{
user.RaiseMessage("Cannot find remote and ID in vref: {0}, halting.", vref);
return(EXIT_ERROR);
}
}
}
// Fix our version string, if required.
metadata = FixVersionStrings(metadata);
// Re-inflate our mod, in case our vref or FixVersionString routines have
// altered it at all.
mod = CkanModule.FromJson(metadata.ToString());
// All done! Write it out!
var version_filename = mod.version.ToString().Replace(':', '-');
string final_path = Path.Combine(options.OutputDir, string.Format("{0}-{1}.ckan", mod.identifier, version_filename));
log.InfoFormat("Writing final metadata to {0}", final_path);
StringBuilder sb = new StringBuilder();
StringWriter sw = new StringWriter(sb);
using (JsonTextWriter writer = new JsonTextWriter(sw))
{
writer.Formatting = Formatting.Indented;
writer.Indentation = 4;
writer.IndentChar = ' ';
JsonSerializer serializer = new JsonSerializer();
serializer.Serialize(writer, metadata);
}
File.WriteAllText(final_path, sw + Environment.NewLine);
return(EXIT_OK);
}
private const string version_token = "$vref"; // It'd be nice if we could have repeated $krefs.
public static int Main(string[] args)
{
BasicConfigurator.Configure();
LogManager.GetRepository().Threshold = Level.Error;
var user = new ConsoleUser();
var options = new CmdLineOptions();
Parser.Default.ParseArgumentsStrict(args, options);
if (options.Verbose)
{
LogManager.GetRepository().Threshold = Level.Info;
}
else if (options.Debug)
{
LogManager.GetRepository().Threshold = Level.Debug;
}
if (options.File == null)
{
log.Fatal("Usage: netkan [--verbose|--debug] [--outputdir=...] <filename>");
return EXIT_BADOPT;
}
NetFileCache cache = FindCache(options, new KSPManager(user), user);
log.InfoFormat("Processing {0}", options.File);
JObject json = JsonFromFile(options.File);
NetKanRemote remote = FindRemote(json);
JObject metadata;
switch (remote.source)
{
case "kerbalstuff":
metadata = KerbalStuff(json, remote.id, cache);
break;
case "github":
if (options.GitHubToken != null)
{
GithubAPI.SetCredentials(options.GitHubToken);
}
metadata = GitHub(json, remote.id, cache);
break;
default:
log.FatalFormat("Unknown remote source: {0}", remote.source);
return EXIT_ERROR;
}
if (metadata == null)
{
log.Error("There was an error, aborting");
return EXIT_ERROR;
}
// Make sure that at the very least this validates against our own
// internal model.
CkanModule mod = CkanModule.FromJson(metadata.ToString());
// Make sure our identifiers match.
if (mod.identifier != (string)json["identifier"])
{
log.FatalFormat("Error: Have metadata for {0}, but wanted {1}", mod.identifier, json["identifier"]);
return EXIT_ERROR;
}
// Find our cached file, we'll need it later.
string file = cache.GetCachedZip(mod.download);
if (file == null)
{
log.FatalFormat("Error: Unable to find {0} in the cache", mod.identifier);
return EXIT_ERROR;
}
// Make sure this would actually generate an install
try
{
ModuleInstaller.FindInstallableFiles(mod, file, null);
}
catch (BadMetadataKraken kraken)
{
log.FatalFormat("Error: Bad metadata for {0}: {1}", kraken.module, kraken.Message);
return EXIT_ERROR;
}
// Now inflate our original metadata from AVC, if we have it.
// The reason we inflate our metadata and not the module is that in the module we can't
// tell if there's been an override of the version fields or not.
// TODO: Remove magic string "#/ckan/ksp-avc"
if (metadata[version_token] != null && (string) metadata[version_token] == "#/ckan/ksp-avc")
{
metadata.Remove(version_token);
try {
AVC avc = AVC.FromZipFile(mod, file);
avc.InflateMetadata(metadata, null, null);
}
catch (JsonReaderException)
{
user.RaiseMessage("Bad embedded KSP-AVC file for {0}, halting.", mod);
return EXIT_ERROR;
}
// If we've done this, we need to re-inflate our mod, too.
mod = CkanModule.FromJson(metadata.ToString());
}
// All done! Write it out!
string final_path = Path.Combine(options.OutputDir, String.Format ("{0}-{1}.ckan", mod.identifier, mod.version));
log.InfoFormat("Writing final metadata to {0}", final_path);
File.WriteAllText(final_path, metadata.ToString());
return EXIT_OK;
}
public CachingHttpService(NetFileCache cache)
{
_cache = cache;
}
/// <summary>
/// Fetch things from Github, returning a complete CkanModule document.
/// </summary>
private static JObject GitHub(JObject orig_metadata, string repo, bool prerelease, NetFileCache cache)
{
// Find the release on github and download.
GithubRelease release = GithubAPI.GetLatestRelease(repo, prerelease);
if (release == null)
{
log.Error("Downloaded releases for " + repo + " but there were none");
return null;
}
string filename = release.Download((string) orig_metadata["identifier"], cache);
// Extract embedded metadata, or use what we have.
JObject metadata = MetadataFromFileOrDefault(filename, orig_metadata);
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/github")
{
log.InfoFormat("Inflating from github metadata... {0}", metadata[expand_token]);
release.InflateMetadata(metadata, filename, repo);
metadata.Remove(expand_token);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
return metadata;
}
/// <summary>
/// Fetch le things from le KerbalStuff.
/// Returns a JObject that should be a fully formed CKAN file.
/// </summary>
internal static JObject KerbalStuff(JObject orig_metadata, string remote_id, NetFileCache cache)
{
// Look up our mod on KS by its ID.
KSMod ks = KSAPI.Mod(Convert.ToInt32(remote_id));
KSVersion latest = ks.Latest();
Version version = latest.friendly_version;
log.DebugFormat("Mod: {0} {1}", ks.name, version);
// Find the latest download.
string filename = latest.Download((string) orig_metadata["identifier"], cache);
JObject metadata = MetadataFromFileOrDefault(filename, orig_metadata);
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/kerbalstuff")
{
log.InfoFormat("Inflating from KerbalStuff... {0}", metadata[expand_token]);
ks.InflateMetadata(metadata, filename, latest);
metadata.Remove(expand_token);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
return metadata;
}
/// <summary>
/// Fetch things from Github, returning a complete CkanModule document.
/// </summary>
private static JObject GitHub(JObject orig_metadata, string repo, bool prerelease, NetFileCache cache)
{
// Find the release on github and download.
GithubRelease release = GithubAPI.GetLatestRelease(repo, prerelease);
if (release == null)
{
log.Error("Downloaded releases for " + repo + " but there were none");
return(null);
}
string filename = release.Download((string)orig_metadata["identifier"], cache);
// Extract embedded metadata, or use what we have.
JObject metadata = MetadataFromFileOrDefault(filename, orig_metadata);
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/github")
{
log.InfoFormat("Inflating from github metadata... {0}", metadata[expand_token]);
release.InflateMetadata(metadata, filename, repo);
metadata.Remove(expand_token);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
return(metadata);
}
private string DownloadPackage(Uri url, string identifier, DateTime?updated, Uri primaryUrl = null)
{
if (primaryUrl == null)
{
primaryUrl = url;
}
if (_overwriteCache && !_requestedURLs.Contains(url))
{
// Discard cached file if command line says so,
// but only the first time in each run
_cache.Remove(url);
}
_requestedURLs.Add(url);
var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);
if (!string.IsNullOrWhiteSpace(cachedFile))
{
return(cachedFile);
}
else
{
var downloadedFile = Net.Download(url);
string extension;
switch (FileIdentifier.IdentifyFile(downloadedFile))
{
case FileType.ASCII:
extension = "txt";
break;
case FileType.GZip:
extension = "gz";
break;
case FileType.Tar:
extension = "tar";
break;
case FileType.TarGz:
extension = "tar.gz";
break;
case FileType.Zip:
extension = "zip";
string invalidReason;
if (!NetFileCache.ZipValid(downloadedFile, out invalidReason))
{
log.Debug($"{downloadedFile} is not a valid ZIP file: {invalidReason}");
throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");
}
break;
default:
extension = "ckan-package";
break;
}
return(_cache.Store(
primaryUrl,
downloadedFile,
string.Format("netkan-{0}.{1}", identifier, extension),
move: true
));
}
}
/// <summary>
/// Shows information about the mod.
/// </summary>
/// <returns>Success status.</returns>
/// <param name="module">The module to show.</param>
public int ShowMod(Module module)
{
#region Abstract and description
if (!string.IsNullOrEmpty(module.@abstract))
{
user.RaiseMessage("{0}: {1}", module.name, module.@abstract);
}
else
{
user.RaiseMessage("{0}", module.name);
}
if (!string.IsNullOrEmpty(module.description))
{
user.RaiseMessage("\n{0}\n", module.description);
}
#endregion
#region General info (author, version...)
user.RaiseMessage("\nModule info:");
user.RaiseMessage("- version:\t{0}", module.version);
if (module.author != null)
{
user.RaiseMessage("- authors:\t{0}", string.Join(", ", module.author));
}
else
{
// Did you know that authors are optional in the spec?
// You do now. #673.
user.RaiseMessage("- authors:\tUNKNOWN");
}
user.RaiseMessage("- status:\t{0}", module.release_status);
user.RaiseMessage("- license:\t{0}", string.Join(", ", module.license));
#endregion
#region Relationships
if (module.depends != null && module.depends.Count > 0)
{
user.RaiseMessage("\nDepends:");
foreach (RelationshipDescriptor dep in module.depends)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.recommends != null && module.recommends.Count > 0)
{
user.RaiseMessage("\nRecommends:");
foreach (RelationshipDescriptor dep in module.recommends)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.suggests != null && module.suggests.Count > 0)
{
user.RaiseMessage("\nSuggests:");
foreach (RelationshipDescriptor dep in module.suggests)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.ProvidesList != null && module.ProvidesList.Count > 0)
{
user.RaiseMessage("\nProvides:");
foreach (string prov in module.ProvidesList)
{
user.RaiseMessage("- {0}", prov);
}
}
#endregion
user.RaiseMessage("\nResources:");
if (module.resources != null)
{
if (module.resources.bugtracker != null)
{
user.RaiseMessage("- bugtracker: {0}", Uri.EscapeUriString(module.resources.bugtracker.ToString()));
}
if (module.resources.homepage != null)
{
user.RaiseMessage("- homepage: {0}", Uri.EscapeUriString(module.resources.homepage.ToString()));
}
if (module.resources.kerbalstuff != null)
{
user.RaiseMessage("- kerbalstuff: {0}", Uri.EscapeUriString(module.resources.kerbalstuff.ToString()));
}
if (module.resources.repository != null)
{
user.RaiseMessage("- repository: {0}", Uri.EscapeUriString(module.resources.repository.ToString()));
}
}
// Compute the CKAN filename.
string file_uri_hash = NetFileCache.CreateURLHash(module.download);
string file_name = CkanModule.StandardName(module.identifier, module.version);
user.RaiseMessage("\nFilename: {0}", file_uri_hash + "-" + file_name);
return(Exit.OK);
}
public void MakeCache()
{
Directory.CreateDirectory(cache_dir);
cache = new NetFileCache(cache_dir);
}
public void MakeCache()
{
cache_dir = TestData.NewTempDir();
Directory.CreateDirectory(cache_dir);
cache = new NetFileCache(cache_dir);
}
public CachingHttpService(NetFileCache cache)
{
_cache = cache;
}
public void TestFixtureTearDown()
{
_cache.Dispose();
_cache = null;
Directory.Delete(_cachePath, recursive: true);
}
public CachingHttpService(NetFileCache cache, bool overwrite = false)
{
_cache = cache;
_overwriteCache = overwrite;
}
internal static JObject HTTP(JObject orig_metadata, string remote_id, NetFileCache cache, IUser user)
{
var metadata = orig_metadata;
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/http")
{
log.InfoFormat("Inflating from HTTP download... {0}", metadata[expand_token]);
metadata["download"] = remote_id;
metadata["version"] = "0.0.0"; // add a dummy version that will be replaced by the KSP-AVC parser later
metadata.Remove(expand_token);
var remote_uri = new Uri(remote_id);
var downloaded_file = Net.Download(remote_uri);
var module = CkanModule.FromJson(metadata.ToString());
// Check the type of the downloaded file.
FileType downloaded_file_type = FileIdentifier.IdentifyFile(downloaded_file);
if (downloaded_file_type != FileType.Zip)
{
// We assume the downloaded file is a zip file later on.
string error_message = String.Format("Downloaded file not identified as a zip. Got {0} instead.", downloaded_file_type.ToString());
throw new Kraken(error_message);
}
if (metadata[version_token] != null && (metadata[version_token].ToString()).StartsWith("#/ckan/ksp-avc"))
{
// TODO pass the correct vref here...
var versionRemote = FindVersionRemote(metadata, metadata[version_token].ToString());
metadata.Remove(version_token);
try
{
AVC avc = AVC.FromZipFile(module, downloaded_file, versionRemote.id);
avc.InflateMetadata(metadata, null, null);
metadata["version"] = avc.version.ToString();
module.version = avc.version;
}
catch (JsonReaderException)
{
user.RaiseMessage("Bad embedded KSP-AVC file for {0}, halting.", module);
return null;
}
// If we've done this, we need to re-inflate our mod, too.
module = CkanModule.FromJson(metadata.ToString());
}
else
{
throw new Kraken("No $vref specified, $kref HTTP method requires it, bailing out..");
}
ModuleInstaller.CachedOrDownload(module.identifier, module.version, module.download, cache);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
// metadata["download"] = metadata["download"].ToString() + '#' + metadata["version"].ToString();
return metadata;
}
/// <summary>
/// Shows information about the mod.
/// </summary>
/// <returns>Success status.</returns>
/// <param name="module">The module to show.</param>
public int ShowMod(CfanModule module)
{
#region Abstract and description
user.RaiseMessage("{0}", module.title);
if (!string.IsNullOrEmpty(module.description))
{
user.RaiseMessage("\n{0}\n", module.description);
}
#endregion
#region General info (author, version...)
user.RaiseMessage("\nModule info:");
user.RaiseMessage("- version:\t{0}", module.modVersion);
if (module.authors != null)
{
user.RaiseMessage("- authors:\t{0}", string.Join(", ", module.authors));
}
else
{
// Did you know that authors are optional in the spec?
// You do now. #673.
user.RaiseMessage("- authors:\tUNKNOWN");
}
#endregion
#region Relationships
if (module.depends != null && module.depends.Any())
{
user.RaiseMessage("\nDepends:");
foreach (var dep in module.depends)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.recommends != null && module.recommends.Any())
{
user.RaiseMessage("\nRecommends:");
foreach (var dep in module.recommends)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.suggests != null && module.suggests.Any())
{
user.RaiseMessage("\nSuggests:");
foreach (var dep in module.suggests)
{
user.RaiseMessage("- {0}", RelationshipToPrintableString(dep));
}
}
if (module.providesNames != null && module.providesNames.Any())
{
user.RaiseMessage("\nProvides:");
foreach (string prov in module.providesNames)
{
user.RaiseMessage("- {0}", prov);
}
}
#endregion
user.RaiseMessage("\nResources:");
if (!String.IsNullOrEmpty(module.homepage))
{
user.RaiseMessage("- homepage: {0}", Uri.EscapeUriString(module.homepage));
}
if (!String.IsNullOrEmpty(module.contact))
{
user.RaiseMessage("- contact: {0}", Uri.EscapeUriString(module.contact));
}
if (!module.isMetapackage)
{
// Compute the CKAN filename.
string file_uri_hash = NetFileCache.CreateURLHash(module.download);
string file_name = CfanModule.createStandardFileName(module.identifier, module.modVersion.ToString());
user.RaiseMessage("\nFilename: {0}", file_uri_hash + "-" + file_name);
}
return(Exit.OK);
}
/// <summary>
/// Fetch Metadata from (successful) Jenkins builds
/// Returns a JObject that should be a fully formed CKAN file.
/// </summary>
internal static JObject Jenkins(JObject orig_metadata, string remote_id, NetFileCache cache)
{
string versionBase = (string) orig_metadata ["x_ci_version_base"];
log.DebugFormat ("versionBase: {0}", versionBase);
JObject resources = (JObject) orig_metadata ["resources"];
log.DebugFormat ("resources: {0}", resources);
string baseUri = (string) resources ["ci"];
if (baseUri == null)
{
// Fallback, we don't have the defined resource 'ci' in the schema yet...
baseUri = (string) resources ["x_ci"];
}
log.DebugFormat ("baseUri: {0}", baseUri);
JenkinsBuild build = JenkinsAPI.GetLatestBuild (baseUri, versionBase, true);
Version version = build.version;
log.DebugFormat("Mod: {0} {1}", remote_id, version);
// Find the latest download.
string filename = build.Download((string) orig_metadata["identifier"], cache);
JObject metadata = MetadataFromFileOrDefault(filename, orig_metadata);
// Check if we should auto-inflate.
string kref = (string)metadata[expand_token];
if (kref == (string)orig_metadata[expand_token] || kref == "#/ckan/kerbalstuff")
{
log.InfoFormat("Inflating from Jenkins... {0}", metadata[expand_token]);
build.InflateMetadata(metadata, filename, null);
metadata.Remove(expand_token);
}
else
{
log.WarnFormat("Not inflating metadata for {0}", orig_metadata["identifier"]);
}
return metadata;
}
