private bool NameCRCEquals(Int32 NameCRC) { var mstr = new MemoryStream(_FileNameBytes, false); var crc = new Ionic.Crc.CRC32(); return(NameCRC == crc.GetCrc32(mstr)); }
private static Int32 GetCrc(string fname) { using (var fs1 = File.OpenRead(fname)) { var checker = new Ionic.Crc.CRC32(true); return(checker.GetCrc32(fs1)); } }
private static Int32 GetCrc(string fname) { using (var fs1 = File.OpenRead(fname)) { var checker = new Ionic.Crc.CRC32(true); return checker.GetCrc32(fs1); } }
private Int32 FigureCrc32() { if (_crcCalculated == false) { Stream input = null; // get the original stream: if (this._Source == ZipEntrySource.WriteDelegate) { var output = new Ionic.Crc.CrcCalculatorStream(Stream.Null); // allow the application to write the data this._WriteDelegate(this.FileName, output); _Crc32 = output.Crc; } else if (this._Source == ZipEntrySource.ZipFile) { // nothing to do - the CRC is already set } else { if (this._Source == ZipEntrySource.Stream) { PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.JitStream) { // allow the application to open the stream if (this._sourceStream == null) _sourceStream = this._OpenDelegate(this.FileName); PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.ZipOutputStream) { } else { //input = File.OpenRead(LocalFileName); input = File.Open(LocalFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } var crc32 = new Ionic.Crc.CRC32(); _Crc32 = crc32.GetCrc32(input); if (_sourceStream == null) { #if NETCF input.Close(); #else input.Dispose(); #endif } } _crcCalculated = true; } return _Crc32; }
public void Zlib_ParallelDeflateStream() { var sw = new System.Diagnostics.Stopwatch(); sw.Start(); TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream Start", sw.Elapsed); int sz = 256*1024 + this.rnd.Next(120000); string FileToCompress = System.IO.Path.Combine(TopLevelDir, String.Format("Zlib_ParallelDeflateStream.{0}.txt", sz)); CreateAndFillFileText( FileToCompress, sz); TestContext.WriteLine("{0}: Created file: {1}", sw.Elapsed, FileToCompress ); byte[] original = File.ReadAllBytes(FileToCompress); int crc1 = DoCrc(FileToCompress); TestContext.WriteLine("{0}: Original CRC: {1:X8}", sw.Elapsed, crc1 ); byte[] working = new byte[WORKING_BUFFER_SIZE]; int n = -1; long originalLength; MemoryStream ms1 = new MemoryStream(); { using (FileStream fs1 = File.OpenRead(FileToCompress)) { originalLength = fs1.Length; using (var compressor = new Ionic.Zlib.ParallelDeflateOutputStream(ms1, true)) { while ((n = fs1.Read(working, 0, working.Length)) != 0) { compressor.Write(working, 0, n); } } } ms1.Seek(0, SeekOrigin.Begin); } TestContext.WriteLine("{0}: Compressed {1} bytes into {2} bytes", sw.Elapsed, originalLength, ms1.Length); var crc = new Ionic.Crc.CRC32(); int crc2= 0; byte[] decompressedBytes= null; using (MemoryStream ms2 = new MemoryStream()) { using (var decompressor = new DeflateStream(ms1, CompressionMode.Decompress, false)) { while ((n = decompressor.Read(working, 0, working.Length)) != 0) { ms2.Write(working, 0, n); } } TestContext.WriteLine("{0}: Decompressed", sw.Elapsed); TestContext.WriteLine("{0}: Decompressed length: {1}", sw.Elapsed, ms2.Length); ms2.Seek(0, SeekOrigin.Begin); crc2 = crc.GetCrc32(ms2); decompressedBytes = ms2.ToArray(); TestContext.WriteLine("{0}: Decompressed CRC: {1:X8}", sw.Elapsed, crc2 ); } TestContext.WriteLine("{0}: Checking...", sw.Elapsed ); bool check = true; if (originalLength != decompressedBytes.Length) { TestContext.WriteLine("Different lengths."); check = false; } else { for (int i = 0; i < decompressedBytes.Length; i++) { if (original[i] != decompressedBytes[i]) { TestContext.WriteLine("byte {0} differs", i); check = false; break; } } } Assert.IsTrue(check,"Data check failed"); TestContext.WriteLine("{0}: Done...", sw.Elapsed ); }
static void Main(string[] args) { //args = new string[4]; //args[0] = "Mob-WorldData.wad"; //args[1] = "-x"; //args[2] = "*"; //args[3] = "mobout"; System.Diagnostics.Stopwatch MainTimer = new System.Diagnostics.Stopwatch(); MainTimer.Start(); string wad = ""; //wad filename string mode = ""; //Operating mode string arg1 = ""; //Argument 1 for mode string arg2 = ""; //Argument 2 for mode bool HadArgs = false; //Whether the user supplied arguments, or just a filename //Try grabbing wad name and mode try { wad = args[0]; } catch //If the wad/mode couldn't be grabbed { Console.WriteLine("Please specify wad!"); //Print error message PrintHelp(); //Print usage info } try { mode = args[1]; HadArgs = true; } catch //If the wad/mode couldn't be grabbed { Console.WriteLine("No mode selected. Defaulting to extract all."); //Print error message mode = "-x"; //PrintHelp(); //Print usage info } //If the user specified a mode, try grabbing arguments for specified mode if (HadArgs) { try { if (mode == "-x") //If extract mode, grab two arguments { arg1 = args[2]; arg2 = args[3]; } else if (mode == "-r" || mode == "-c" || mode == "-d" || mode == "-a") //Remove/Create/Diff/Add mode, one arg { arg1 = args[2]; } else if (mode != "-i" && mode != "-w2z") //If the mode is not -i (takes no arguments), then we don't know what mode they specified { Console.WriteLine("Invalid mode!"); //Print error PrintHelp(); //Print usage info } } catch //If the arguments were missing, or something else went wrong { Console.WriteLine("Invalid arguments for specified mode!"); //Print an error PrintHelp(); //Print usage info } } else { if (!System.IO.File.Exists(wad)) { Console.WriteLine("Wad file not found!"); PrintHelp(); } arg1 = "*"; //Set extraction file-selection to all System.Windows.Forms.FolderBrowserDialog fd = new System.Windows.Forms.FolderBrowserDialog(); fd.Description = "Choose where to save the extracted files"; MainTimer.Stop(); if (fd.ShowDialog() == System.Windows.Forms.DialogResult.OK) { MainTimer.Start(); arg2 = fd.SelectedPath; ExclusionDir = arg2; } else { Quit(); } } FileList[] entries = new FileList[0]; //Pre-init the 'entries' array if (mode == "-a") //If using file-add mode { try { arg2 = args[3]; } catch { Console.WriteLine("No output wad specified. Overwriting original"); arg2 = wad; } string[] InFiles = arg1.Split(':'); for (int i = 0; i < InFiles.Length; i++) { if (!File.Exists(InFiles[i])) { if (!Directory.Exists(InFiles[i])) { Console.WriteLine("{0} could not be found!\nCancelling operation...", InFiles[i]); Quit(); } else { Console.WriteLine("{0} is a directory! Processing sub-entries...", InFiles[i]); } } else { Console.WriteLine("{0} added", InFiles[i]); } } Quit(); } if (mode == "-c") //Create (wad) mode { //Make sure the input directory exists if (!Directory.Exists(arg1)) { Console.WriteLine("Input directory not found!"); PrintHelp(); } string[] InFiles = Directory.GetFiles(arg1, "*.*", SearchOption.AllDirectories); //Grab a list of all files within the directory entries = new FileList[InFiles.Count()]; //Make a new FileList for storing these files in the wad Console.WriteLine("Filecount: {0}", InFiles.Count()); //Debug //For each file Parallel.For(0, InFiles.Count(), i => { entries[i].Filename = InFiles[i].Substring(arg1.Length, InFiles[i].Length - arg1.Length); //Remove directory info that shouldn't be included in the wad (path up to the wad contents) if (entries[i].Filename.IndexOf("\\") == 0) //If the entry starts with a \ { entries[i].Filename = entries[i].Filename.Substring(1, entries[i].Filename.Length - 1); //Get rid of the slash } entries[i].Filename = entries[i].Filename.Replace('\\', '/'); //MemoryStream ms = new MemoryStream(File.ReadAllBytes(InFiles[i])); //Read entries[i].Data = File.ReadAllBytes(InFiles[i]); //Read the file into memory entries[i].Size = (uint)entries[i].Data.Length; //Console.WriteLine("{0}, with a size of {1}, was read to memory", entries[i].Filename, entries[i].Size); }); Console.WriteLine("Files read to memory"); List <byte> WadHeader = new List <byte>(); List <byte> WadBody = new List <byte>(); var crcparam = new CrcSharp.CrcParameters(32, 0x04c11db7, 0, 0, true, true); var crc = new CrcSharp.Crc(crcparam); for (int i = 0; i < entries.Length; i++) { entries[i].Offset = (uint)WadBody.Count(); //Save the offset for this entry MemoryStream Compressed = new MemoryStream(); Ionic.Zlib.ZlibStream zls = new Ionic.Zlib.ZlibStream(Compressed, Ionic.Zlib.CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, false); zls.Write(entries[i].Data, 0, entries[i].Data.Length); zls.Close(); byte[] compdata = Compressed.ToArray(); WadBody.AddRange(compdata); //Add the compressed data to the body of the wad entries[i].CRC = (uint)crc.CalculateAsNumeric(entries[i].Data); //Add the CRC of the compressed data (idk why they do this) entries[i].CompressedSize = (uint)compdata.Length; //Save the size of the compressed data //Console.WriteLine("CRC: {0} : {1}", entries[i].CRC.ToString("X8"), entries[i].Filename); //Debug } Console.WriteLine("Wad body added"); //Add wad header info WadHeader.AddRange(new byte[] { 0x4B, 0x49, 0x57, 0x41, 0x44, 0x02, 0x00, 0x00, 0x00 }); //Add magic and wad version (2) WadHeader.AddRange(BitConverter.GetBytes(entries.Length)); //Add the file count WadHeader.Add(0x01); //Add wad2 byte (idk what it is, but it's required) int[] offoff = new int[entries.Length]; //Keeps track of the offset for the 'Offset' field in the header (we can't know the offset until the header is finished, so we add it later) //Add each file to the wad header for (int i = 0; i < entries.Length; i++) { offoff[i] = WadHeader.Count(); //Save current offset, so that we can update the 'offset' field when the header is finished WadHeader.AddRange(new byte[] { 0x00, 0x00, 0x00, 0x00 }); //Add filler offset WadHeader.AddRange(BitConverter.GetBytes(entries[i].Size)); //Add uncompresses file size WadHeader.AddRange(BitConverter.GetBytes(entries[i].CompressedSize)); //Add compressed file size WadHeader.Add(0x01); //Set 'IsCompressed' to 1 (we compress all files, so this is always true) WadHeader.AddRange(BitConverter.GetBytes(entries[i].CRC)); //Add the compressed-data crc (idk why they have this, because the extracted file has a crc anyway) WadHeader.AddRange(BitConverter.GetBytes(entries[i].Filename.Length + 1)); //Add filename length WadHeader.AddRange(ASCIIEncoding.ASCII.GetBytes(entries[i].Filename)); //Add the filename WadHeader.Add(0x00); //Add padding } Console.WriteLine("Wad Header added"); //Fix the offsets for (int i = 0; i < entries.Length; i++) { byte[] offsetbytes = BitConverter.GetBytes(WadHeader.Count() + entries[i].Offset); WadHeader[offoff[i]] = offsetbytes[0]; WadHeader[offoff[i] + 1] = offsetbytes[1]; WadHeader[offoff[i] + 2] = offsetbytes[2]; WadHeader[offoff[i] + 3] = offsetbytes[3]; } Console.WriteLine("Offsets fixed"); byte[] Output = WadHeader.Concat(WadBody).ToArray(); File.WriteAllBytes(wad, Output); //Save Console.WriteLine("Wad Saved"); Quit(); } if (mode != "-c") //If the tool is not in create mode, check if the wad exists { if (!System.IO.File.Exists(wad)) { Console.WriteLine("Wad file not found!"); PrintHelp(); } else //If the file exists { Console.WriteLine("Reading wad to memory..."); entries = ReadWad(wad); //Read the wad into the 'entries' array } } if (mode == "-d" && !File.Exists(arg1)) //If using diff mode, and the second file doesn't exist { Console.WriteLine("Second wad does not exist!"); PrintHelp(); } if (mode == "-w2z") //If using Wad2Zip mode { //Stopwatch for diagnostics System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); Console.WriteLine("Calculating CRC's..."); //For each file in the wad; extract it in-memory, calculate the CRC of the extracted data, and update the entry's CRC field (because this is done in-memory, it should be less than a couple of seconds) Parallel.For(0, entries.Length, i => { byte[] filemem = new byte[0]; if (entries[i].IsCompressed) //If the file is marked as compressed { filemem = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data); Array.Copy(entries[i].Data, 2, entries[i].Data, 0, entries[i].Data.Length - 6); } else //If the file isn't compressed { filemem = entries[i].Data; } Ionic.Crc.CRC32 crc = new Ionic.Crc.CRC32(); entries[i].CRC = (uint)crc.GetCrc32(new MemoryStream(filemem)); //Replace the entries' crc with the CRC of the compressed data (KI are shit and use their own incompatible polynomials for their checksum, so we need to recalculate it with a standard polynomial) }); stopwatch.Stop(); //Stopwatch for timing the zip-process System.Diagnostics.Stopwatch ziptimer = new System.Diagnostics.Stopwatch(); ziptimer.Start(); byte[] outputfile = Zipper(entries); //Add all file entries to a zip in-memory, and return the zip ziptimer.Stop(); File.WriteAllBytes(wad + ".zip", outputfile); //Save the created zip to disk (input filename with .zip appended) MainTimer.Stop(); Console.WriteLine("Updated CRC's in {0} Seconds", stopwatch.Elapsed.TotalSeconds); Console.WriteLine("Zipped in {0} Seconds", ziptimer.Elapsed.TotalSeconds); Console.WriteLine("Total program runtime: {0} Seconds", MainTimer.Elapsed.TotalSeconds); Quit(); //Exit } if (mode == "-i" || mode == "-x" || mode == "-d") { if (mode == "-i") //If using info mode { StringBuilder sb = new StringBuilder(); //Make a new stringbuilder (stringbuilder is much faster than just appending strings normally) for (int i = 0; i < entries.Length; i++) //For each file entry { sb.AppendLine(entries[i].Offset.ToString("X") + ":" + entries[i].CompressedSize + ":" + entries[i].Size + ":" + entries[i].Filename); //Add a newline to the output string, with the offset and filename (offset:filename) } Console.WriteLine(sb); //Print the output string MainTimer.Stop(); Console.WriteLine("{0} files found in {1} Seconds", entries.Length, MainTimer.Elapsed.TotalSeconds); Quit(); //Quit } else if (mode == "-d") //If using diff mode { Console.WriteLine("Diff-checking {0} and {1}", wad, arg1); bool extract = false; try //Try grabbing the output-folder argument { arg2 = args[3]; //Grab fourth argument (starting from 0) try { arg2 = ResolveDir(arg2); extract = true; //Enable file-extraction } catch //If something went wrong whlie creating the directory { Console.WriteLine("Error creating directory: {0}\nMaybe you're trying to write to a folder you don't have permission to write in?", arg2); //Let the user know something went wrong Quit(); //Exit } } catch //If there isn't an output-folder argument { Console.WriteLine("No output folder specified. No extraction will be performed."); //Print a message stating operating mode } Console.WriteLine("Checking differences..."); FileList[] entries2 = ReadWad(arg1); bool[] ExtractIt = new bool[entries2.Length]; //Create a bool array, which keeps track of which files to extract from the second wad StringBuilder MissingIn1 = new StringBuilder(); StringBuilder MissingIn2 = new StringBuilder(); StringBuilder DiffIn2 = new StringBuilder(); object threadsync = new object(); //Threadsync is used to prevent threads from overwriting eachothers data //Check what files are missing Parallel.For(0, entries.Length, i => //For each file entry in the first wad { bool Exists = false; //Mark the file as not existing in both wads (default, until proven otherwise) for (int j = 0; j < entries2.Length; j++) //For each file in second wad { if (entries2[j].Filename == entries[i].Filename) //If the currently-processes filename in wad 1 is also the same filename in wad2 { Exists = true; //Mark the file as existing in both if (entries2[j].CRC != entries[i].CRC) //If the files have a different CRC { lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses { DiffIn2.AppendLine(entries[i].Filename); //Add the filename to the 'DiffIn2' sb, so we can print the results later } ExtractIt[j] = true; //Mark the file in wad2 for extraction, because it's different } break; //Stop searching for this file, because we've already checked it } } if (!Exists) //If the file wasn't marked as existing { lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses { MissingIn2.AppendLine(entries[i].Filename); //Add the file to the 'MissingIn2' sb for printing later } } }); //Check what files exist in both wads (and whether any files were added) Parallel.For(0, entries2.Length, i => { bool Exists = false; for (int j = 0; j < entries.Length; j++) { if (entries2[i].Filename == entries[j].Filename) { Exists = true; //Mark the file as existing in both break; //Stop searching for this file } } if (!Exists) //If the file wasn't marked as existing { lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses { MissingIn1.AppendLine(entries[i].Filename); //Add the file to the 'MissingIn1' sb for printing later } ExtractIt[i] = true; //Mark the file in wad2 for extraction, because it doesn't exist in wad1 } }); Console.WriteLine("Diff-checking complete!"); if (extract) //If the user specified an output directory (extract diff files) { Console.WriteLine("Extracting new/different files..."); //Inform the user that the files will now be extracted Console.WriteLine("Pre-creating directories..."); for (int i = 0; i < entries2.Length; i++) //For each file in the second wad { if (ExtractIt[i]) //If the file is marked for extraction, check it for any subdirectories, and create them if necessary { PreCreate(entries2[i], arg2); //Call 'PreCreate' for that file, and any required subdirs for that file will be created } } Console.WriteLine("Directories created!\nExtracting..."); //For each file in the second wad, check if it's marked for extraction; and if so, extract it. Parallel.For(0, entries2.Length, i => { if (ExtractIt[i]) //If the file was marked for extraction (new/different file) { if (entries2[i].IsCompressed) //If the file is marked as compressed { entries2[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries2[i].Data); //Decompress the file } using (FileStream output = new FileStream(arg2 + "\\" + entries2[i].Filename, FileMode.Create)) //Create the file that is being extracted (replaces old files if they exist) { output.Write(entries2[i].Data, 0, entries2[i].Data.Length); //Write the file from memory to disk } } }); } if (MissingIn1.Length > 0 || MissingIn2.Length > 0 || DiffIn2.Length > 0) //If there were any file differences { Console.WriteLine("Differences found!"); if (entries.Length != entries2.Length) { Console.WriteLine("------------------------------File count is different!------------------------------\nOld:{0}\tNew:{1}", entries.Length, entries2.Length); } if (MissingIn1.Length > 0) { Console.WriteLine("------------------------------Files missing in first wad------------------------------\n{0}", MissingIn1); } if (MissingIn2.Length > 0) { Console.WriteLine("------------------------------Files missing in second wad------------------------------\n{0}", MissingIn2); } if (DiffIn2.Length > 0) { Console.WriteLine("------------------------------Files changed------------------------------\n{0}", DiffIn2); } } else { Console.WriteLine("No differences found!"); } MainTimer.Stop(); Console.WriteLine("Total program runtime: {0} Seconds", MainTimer.Elapsed.TotalSeconds); Quit(); } else if (mode == "-x") //If using extract mode { arg2 = ResolveDir(arg2); if (arg1 != "*") //If the user specified a file to extract (not all files) { for (int i = 0; i < entries.Length; i++) //For each file in the filelist { if (string.Equals(entries[i].Filename, arg1, StringComparison.OrdinalIgnoreCase) || string.Equals(entries[i].Filename.Replace('/', '\\'), arg1, StringComparison.OrdinalIgnoreCase)) //If the file entry matches the user-specified file (ignoring case and slash-direction) { Console.WriteLine("File found!"); PreCreate(entries[i], arg2); //Check if file is located in a subdirectory. If so, create the appropriate directory structure if (entries[i].IsCompressed) //If the file is marked as compressed { entries[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data); //Decompress the data } using (FileStream output = new FileStream(arg2 + "\\" + entries[i].Filename, FileMode.Create)) //Create the file that is being extracted (replaces old files if they exist) { output.Write(entries[i].Data, 0, entries[i].Data.Length); //Write the file from memory to disk Console.WriteLine("File extracted to: {0}", arg2 + "\\" + entries[i].Filename.Replace('/', '\\')); } Quit(); } //If the filename doesn't match, read the next entry } //If all files have been scanned, and the user-specified file wasn't found; let the user know, and give them some advice Console.WriteLine("'{0}' was not found in the specified wad!", arg1); Console.WriteLine("Make sure you include the file's parent directories"); Console.WriteLine("eg: capabilities\\cpu.xml"); Quit(); } else //If the file is '*": Don't 'really' need to say else here, because it would have exited previously anyway, but it just makes things more readable { //Create directories in advance, using a single thread (prevents race crash when parallel threads attempt to create the same directory at the same time) Console.WriteLine("Pre-creating directories..."); for (int i = 0; i < entries.Length; i++) { PreCreate(entries[i], arg2); //Pre-create any subdirectories listed in the filename (arg2 is the base directory for extraction) } Console.WriteLine("Directories created!\nExtracting..."); System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); Parallel.For(0, entries.Length, i => { if (entries[i].IsCompressed) //If the file is marked as compressed { entries[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data); } }); stopwatch.Stop(); Console.WriteLine("Extraction complete!\nWriting to disk... (this may take some time)"); System.Diagnostics.Stopwatch writetimer = new System.Diagnostics.Stopwatch(); writetimer.Start(); Parallel.For(0, entries.Length, i => { File.WriteAllBytes(arg2 + "\\" + entries[i].Filename, entries[i].Data); }); writetimer.Stop(); MainTimer.Stop(); Console.WriteLine("Extracted {0} files in {1} Seconds", entries.Length, stopwatch.Elapsed.TotalSeconds); Console.WriteLine("Wrote files in {0} Ms", writetimer.ElapsedMilliseconds); Console.WriteLine("Total program runtime: {0} seconds", MainTimer.Elapsed.TotalSeconds); Quit(); //Exit } } } }
Int32 FigureCrc32() { if (_crcCalculated == false) { Stream input = null; // get the original stream: if (this._Source == ZipEntrySource.WriteDelegate) { var output = new Ionic.Crc.CrcCalculatorStream(Stream.Null); // allow the application to write the data this._WriteDelegate(this.FileName, output); _Crc32 = output.Crc; } else if (this._Source == ZipEntrySource.ZipFile) { // nothing to do - the CRC is already set } else { if (this._Source == ZipEntrySource.Stream) { PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.JitStream) { // allow the application to open the stream if (this._sourceStream == null) _sourceStream = this._OpenDelegate(this.FileName); PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.ZipOutputStream) { } else { } var crc32 = new Ionic.Crc.CRC32(); _Crc32 = crc32.GetCrc32(input); } _crcCalculated = true; } return _Crc32; }