public override int Run() { using var sourceCache = this.ArgumentParser.SourceCache; if (sourceCache == null) { Console.WriteLine("No cache source specified."); return(Program.ExitCodeInvalidArgument); } if (this.ArgumentParser.FileFilter == null || this.ArgumentParser.FileFilter.Item1.Length == 0) { Console.WriteLine("No files to extract specified."); return(Program.ExitCodeInvalidArgument); } using var outputCache = new FlatFileCache(this.ArgumentParser.OutputDirectory ?? "files") { OverwriteFiles = !this._preserve, }; foreach (var index in this.ArgumentParser.FileFilter.Item1) { var fileIds = this.ArgumentParser.FileFilter.Item2.Length > 0 ? this.ArgumentParser.FileFilter.Item2 : sourceCache.GetAvailableFileIds(index); Parallel.ForEach(fileIds, (fileId) => { try { var file = sourceCache.GetFile(index, fileId); outputCache.PutFile(index, fileId, file); Log.Information($"File {(int)index}/{fileId}: Extracted."); } catch (CacheFileNotFoundException exception) { Log.Information($"File {(int)index}/{fileId}: {exception.Message}"); } }); } Console.WriteLine("Extraction completed."); return(Program.ExitCodeOk); }
private void Extract() { var outputCache = new FlatFileCache(this._argumentParser.OutputDirectory + "files/"); // // Display progress at bottom of console without creating a new row // var progress = new ExtendedProgress(); // progress.ProgressChanged += (p, message) => // { // Console.Write($"Extraction progress: {Math.Round(progress.Percentage)}% ({progress.Current}/{progress.Total})\r"); // }; if (this._argumentParser.Indexes == null && this._argumentParser.FileIds != null) { throw new ArgumentException("If you specify files to extract you must also explicitly specify indexes to extract."); } foreach (var index in this._argumentParser.Indexes ?? this._cache.GetIndexes()) { // Create a list of files to be extracted (requested if overwriting, missing if not) var requestedFileIds = this._argumentParser.FileIds ?? this._cache.GetFileIds(index); var existingFileIds = outputCache.GetFileIds(index).ToList(); var fileIds = this._argumentParser.Overwrite ? requestedFileIds : requestedFileIds.Where(fileId => !existingFileIds.Contains(fileId)); Parallel.ForEach( fileIds, new ParallelOptions { MaxDegreeOfParallelism = 10, }, fileId => { this._cache.CopyFile(index, fileId, outputCache); } ); } }
public void ConstructTestCache() { var files = new List <Tuple <CacheIndex, int> > { new Tuple <CacheIndex, int>(CacheIndex.Enums, 5), new Tuple <CacheIndex, int>(CacheIndex.ClientScripts, 3), new Tuple <CacheIndex, int>(CacheIndex.LoadingSprites, 30462), new Tuple <CacheIndex, int>(CacheIndex.Models, 47000), new Tuple <CacheIndex, int>(CacheIndex.Enums, 23), new Tuple <CacheIndex, int>(CacheIndex.ItemDefinitions, 155), new Tuple <CacheIndex, int>(CacheIndex.ItemDefinitions, 5), new Tuple <CacheIndex, int>(CacheIndex.AnimationFrames, 0), // Soundscape new Tuple <CacheIndex, int>(CacheIndex.Music, 38900), new Tuple <CacheIndex, int>(CacheIndex.Music, 73450), new Tuple <CacheIndex, int>(CacheIndex.Music, 73451), new Tuple <CacheIndex, int>(CacheIndex.Music, 73452), new Tuple <CacheIndex, int>(CacheIndex.Music, 73453), new Tuple <CacheIndex, int>(CacheIndex.Music, 73454), new Tuple <CacheIndex, int>(CacheIndex.Music, 73455), new Tuple <CacheIndex, int>(CacheIndex.Music, 73456), new Tuple <CacheIndex, int>(CacheIndex.Music, 73457), new Tuple <CacheIndex, int>(CacheIndex.Music, 73458), new Tuple <CacheIndex, int>(CacheIndex.Music, 73459), new Tuple <CacheIndex, int>(CacheIndex.Music, 73460), new Tuple <CacheIndex, int>(CacheIndex.Music, 73461), new Tuple <CacheIndex, int>(CacheIndex.Music, 73462), new Tuple <CacheIndex, int>(CacheIndex.Music, 73463), new Tuple <CacheIndex, int>(CacheIndex.Music, 73464), new Tuple <CacheIndex, int>(CacheIndex.Music, 73465), new Tuple <CacheIndex, int>(CacheIndex.Music, 73466), new Tuple <CacheIndex, int>(CacheIndex.Music, 73467), new Tuple <CacheIndex, int>(CacheIndex.Music, 73468), new Tuple <CacheIndex, int>(CacheIndex.Music, 73469), new Tuple <CacheIndex, int>(CacheIndex.Music, 73470), new Tuple <CacheIndex, int>(CacheIndex.Music, 73471), new Tuple <CacheIndex, int>(CacheIndex.Music, 73472), new Tuple <CacheIndex, int>(CacheIndex.Music, 73473), new Tuple <CacheIndex, int>(CacheIndex.Music, 73474), new Tuple <CacheIndex, int>(CacheIndex.Music, 73475), new Tuple <CacheIndex, int>(CacheIndex.Music, 73476), new Tuple <CacheIndex, int>(CacheIndex.Music, 73477), new Tuple <CacheIndex, int>(CacheIndex.Music, 73478), new Tuple <CacheIndex, int>(CacheIndex.Music, 73479), new Tuple <CacheIndex, int>(CacheIndex.Music, 73480), new Tuple <CacheIndex, int>(CacheIndex.Music, 73481), new Tuple <CacheIndex, int>(CacheIndex.Music, 73482), new Tuple <CacheIndex, int>(CacheIndex.Music, 73483), new Tuple <CacheIndex, int>(CacheIndex.Music, 73484), new Tuple <CacheIndex, int>(CacheIndex.Music, 73485), new Tuple <CacheIndex, int>(CacheIndex.Music, 73486), new Tuple <CacheIndex, int>(CacheIndex.Music, 73487), new Tuple <CacheIndex, int>(CacheIndex.Music, 73488), new Tuple <CacheIndex, int>(CacheIndex.Music, 73489), new Tuple <CacheIndex, int>(CacheIndex.Music, 73490), new Tuple <CacheIndex, int>(CacheIndex.Music, 73491), new Tuple <CacheIndex, int>(CacheIndex.Music, 73492), new Tuple <CacheIndex, int>(CacheIndex.Music, 73493), new Tuple <CacheIndex, int>(CacheIndex.Music, 73494), new Tuple <CacheIndex, int>(CacheIndex.Music, 73495), new Tuple <CacheIndex, int>(CacheIndex.Music, 73496), new Tuple <CacheIndex, int>(CacheIndex.Music, 73497), new Tuple <CacheIndex, int>(CacheIndex.Music, 73498), new Tuple <CacheIndex, int>(CacheIndex.Music, 73499), new Tuple <CacheIndex, int>(CacheIndex.Music, 73500), new Tuple <CacheIndex, int>(CacheIndex.Music, 73501), // Black Zabeth: LIVE! new Tuple <CacheIndex, int>(CacheIndex.Music, 4978), new Tuple <CacheIndex, int>(CacheIndex.Music, 45182), new Tuple <CacheIndex, int>(CacheIndex.Music, 45183), new Tuple <CacheIndex, int>(CacheIndex.Music, 45184), new Tuple <CacheIndex, int>(CacheIndex.Music, 45185), new Tuple <CacheIndex, int>(CacheIndex.Music, 45186), new Tuple <CacheIndex, int>(CacheIndex.Music, 45187), new Tuple <CacheIndex, int>(CacheIndex.Music, 45188), new Tuple <CacheIndex, int>(CacheIndex.Music, 45189), new Tuple <CacheIndex, int>(CacheIndex.Music, 45190), new Tuple <CacheIndex, int>(CacheIndex.Music, 45191), new Tuple <CacheIndex, int>(CacheIndex.Music, 45192), new Tuple <CacheIndex, int>(CacheIndex.Music, 45193), new Tuple <CacheIndex, int>(CacheIndex.Music, 45194), new Tuple <CacheIndex, int>(CacheIndex.Music, 45195), new Tuple <CacheIndex, int>(CacheIndex.Music, 45196), new Tuple <CacheIndex, int>(CacheIndex.Music, 45197), new Tuple <CacheIndex, int>(CacheIndex.Music, 45198), new Tuple <CacheIndex, int>(CacheIndex.Music, 45199), new Tuple <CacheIndex, int>(CacheIndex.Music, 45200), new Tuple <CacheIndex, int>(CacheIndex.Music, 45201), new Tuple <CacheIndex, int>(CacheIndex.Music, 45202), new Tuple <CacheIndex, int>(CacheIndex.Music, 45203), new Tuple <CacheIndex, int>(CacheIndex.Music, 45204), new Tuple <CacheIndex, int>(CacheIndex.Music, 45205), new Tuple <CacheIndex, int>(CacheIndex.Music, 45206), new Tuple <CacheIndex, int>(CacheIndex.Music, 45207), new Tuple <CacheIndex, int>(CacheIndex.Music, 45208), new Tuple <CacheIndex, int>(CacheIndex.Music, 45209), new Tuple <CacheIndex, int>(CacheIndex.Music, 45210), new Tuple <CacheIndex, int>(CacheIndex.Music, 45211), new Tuple <CacheIndex, int>(CacheIndex.Music, 45212), new Tuple <CacheIndex, int>(CacheIndex.Music, 45213), new Tuple <CacheIndex, int>(CacheIndex.Music, 45214), new Tuple <CacheIndex, int>(CacheIndex.Music, 45215), new Tuple <CacheIndex, int>(CacheIndex.Music, 45216), new Tuple <CacheIndex, int>(CacheIndex.Music, 45217), new Tuple <CacheIndex, int>(CacheIndex.Music, 45218), new Tuple <CacheIndex, int>(CacheIndex.Music, 45219), }; if (Directory.Exists("generated")) { Directory.Delete("generated", true); } var expectedFileSizes = new Queue <int>(); // Download and write the files. using (var downloaderCache = new DownloaderCache()) using (var flatFileCache = new FlatFileCache("generated/file")) using (var javaCache = new JavaClientCache("generated/java", false)) using (var nxtCache = new NxtClientCache("generated/nxt", false)) { foreach (var fileTuple in files) { var file = downloaderCache.GetFile(fileTuple.Item1, fileTuple.Item2); flatFileCache.PutFile(fileTuple.Item1, fileTuple.Item2, file); javaCache.PutFile(fileTuple.Item1, fileTuple.Item2, file); nxtCache.PutFile(fileTuple.Item1, fileTuple.Item2, file); expectedFileSizes.Enqueue(file.Data.Length); // Note: Encodes entries in RT5 format. } } // Verify that the files are now obtainable and unchanged. using (var flatFileCache = new FlatFileCache("generated/file")) using (var javaCache = new JavaClientCache("generated/java")) using (var nxtCache = new NxtClientCache("generated/nxt")) { foreach (var fileTuple in files) { var expectedFileSize = expectedFileSizes.Dequeue(); var flatFile = flatFileCache.GetFile(fileTuple.Item1, fileTuple.Item2); Assert.Equal(expectedFileSize, flatFile.Data.Length); var javaFile = javaCache.GetFile(fileTuple.Item1, fileTuple.Item2); Assert.Equal(expectedFileSize, javaFile.Data.Length); var nxtFile = nxtCache.GetFile(fileTuple.Item1, fileTuple.Item2); Assert.Equal(expectedFileSize, nxtFile.Data.Length); } } }
public void ConstructTestCache() { var files = new List <Tuple <Index, int> > { new Tuple <Index, int>(Index.Enums, 5), new Tuple <Index, int>(Index.ClientScripts, 3), new Tuple <Index, int>(Index.LoadingSprites, 30556), new Tuple <Index, int>(Index.Models, 47000), new Tuple <Index, int>(Index.Enums, 23), new Tuple <Index, int>(Index.ItemDefinitions, 155), new Tuple <Index, int>(Index.ItemDefinitions, 5), new Tuple <Index, int>(Index.AnimationFrames, 0), // Soundscape new Tuple <Index, int>(Index.Music, 38900), new Tuple <Index, int>(Index.Music, 73450), new Tuple <Index, int>(Index.Music, 73451), new Tuple <Index, int>(Index.Music, 73452), new Tuple <Index, int>(Index.Music, 73453), new Tuple <Index, int>(Index.Music, 73454), new Tuple <Index, int>(Index.Music, 73455), new Tuple <Index, int>(Index.Music, 73456), new Tuple <Index, int>(Index.Music, 73457), new Tuple <Index, int>(Index.Music, 73458), new Tuple <Index, int>(Index.Music, 73459), new Tuple <Index, int>(Index.Music, 73460), new Tuple <Index, int>(Index.Music, 73461), new Tuple <Index, int>(Index.Music, 73462), new Tuple <Index, int>(Index.Music, 73463), new Tuple <Index, int>(Index.Music, 73464), new Tuple <Index, int>(Index.Music, 73465), new Tuple <Index, int>(Index.Music, 73466), new Tuple <Index, int>(Index.Music, 73467), new Tuple <Index, int>(Index.Music, 73468), new Tuple <Index, int>(Index.Music, 73469), new Tuple <Index, int>(Index.Music, 73470), new Tuple <Index, int>(Index.Music, 73471), new Tuple <Index, int>(Index.Music, 73472), new Tuple <Index, int>(Index.Music, 73473), new Tuple <Index, int>(Index.Music, 73474), new Tuple <Index, int>(Index.Music, 73475), new Tuple <Index, int>(Index.Music, 73476), new Tuple <Index, int>(Index.Music, 73477), new Tuple <Index, int>(Index.Music, 73478), new Tuple <Index, int>(Index.Music, 73479), new Tuple <Index, int>(Index.Music, 73480), new Tuple <Index, int>(Index.Music, 73481), new Tuple <Index, int>(Index.Music, 73482), new Tuple <Index, int>(Index.Music, 73483), new Tuple <Index, int>(Index.Music, 73484), new Tuple <Index, int>(Index.Music, 73485), new Tuple <Index, int>(Index.Music, 73486), new Tuple <Index, int>(Index.Music, 73487), new Tuple <Index, int>(Index.Music, 73488), new Tuple <Index, int>(Index.Music, 73489), new Tuple <Index, int>(Index.Music, 73490), new Tuple <Index, int>(Index.Music, 73491), new Tuple <Index, int>(Index.Music, 73492), new Tuple <Index, int>(Index.Music, 73493), new Tuple <Index, int>(Index.Music, 73494), new Tuple <Index, int>(Index.Music, 73495), new Tuple <Index, int>(Index.Music, 73496), new Tuple <Index, int>(Index.Music, 73497), new Tuple <Index, int>(Index.Music, 73498), new Tuple <Index, int>(Index.Music, 73499), new Tuple <Index, int>(Index.Music, 73500), new Tuple <Index, int>(Index.Music, 73501), // Black Zabeth: LIVE! new Tuple <Index, int>(Index.Music, 4978), new Tuple <Index, int>(Index.Music, 45182), new Tuple <Index, int>(Index.Music, 45183), new Tuple <Index, int>(Index.Music, 45184), new Tuple <Index, int>(Index.Music, 45185), new Tuple <Index, int>(Index.Music, 45186), new Tuple <Index, int>(Index.Music, 45187), new Tuple <Index, int>(Index.Music, 45188), new Tuple <Index, int>(Index.Music, 45189), new Tuple <Index, int>(Index.Music, 45190), new Tuple <Index, int>(Index.Music, 45191), new Tuple <Index, int>(Index.Music, 45192), new Tuple <Index, int>(Index.Music, 45193), new Tuple <Index, int>(Index.Music, 45194), new Tuple <Index, int>(Index.Music, 45195), new Tuple <Index, int>(Index.Music, 45196), new Tuple <Index, int>(Index.Music, 45197), new Tuple <Index, int>(Index.Music, 45198), new Tuple <Index, int>(Index.Music, 45199), new Tuple <Index, int>(Index.Music, 45200), new Tuple <Index, int>(Index.Music, 45201), new Tuple <Index, int>(Index.Music, 45202), new Tuple <Index, int>(Index.Music, 45203), new Tuple <Index, int>(Index.Music, 45204), new Tuple <Index, int>(Index.Music, 45205), new Tuple <Index, int>(Index.Music, 45206), new Tuple <Index, int>(Index.Music, 45207), new Tuple <Index, int>(Index.Music, 45208), new Tuple <Index, int>(Index.Music, 45209), new Tuple <Index, int>(Index.Music, 45210), new Tuple <Index, int>(Index.Music, 45211), new Tuple <Index, int>(Index.Music, 45212), new Tuple <Index, int>(Index.Music, 45213), new Tuple <Index, int>(Index.Music, 45214), new Tuple <Index, int>(Index.Music, 45215), new Tuple <Index, int>(Index.Music, 45216), new Tuple <Index, int>(Index.Music, 45217), new Tuple <Index, int>(Index.Music, 45218), new Tuple <Index, int>(Index.Music, 45219) }; try { Directory.Delete("generated", true); } catch (DirectoryNotFoundException) { } // Download and write the files using (var downloader = new DownloaderCache()) using (var runeTek5Cache = new RuneTek5Cache("generated/runetek5", false)) using (var flatFileCache = new FlatFileCache("generated/flatfile")) { foreach (var fileTuple in files) { var file = downloader.GetFile <BinaryFile>(fileTuple.Item1, fileTuple.Item2); runeTek5Cache.PutFile(file); flatFileCache.PutFile(file); } } // Verify that the files are now obtainable using (var freshRuneTek5Cache = new RuneTek5Cache("generated/runetek5", true)) using (var freshFlatFileCache = new FlatFileCache("generated/flatfile")) { foreach (var fileTuple in files) { freshRuneTek5Cache.GetFile <BinaryFile>(fileTuple.Item1, fileTuple.Item2); freshFlatFileCache.GetFile <BinaryFile>(fileTuple.Item1, fileTuple.Item2); } } }
public FlatFileCacheTests(TestCacheFixture fixture, ITestOutputHelper output) { this._fixture = fixture; this._output = output; this._outputFlatFileCache = new FlatFileCache("output"); }