static void Decompress11LZS(string filein, string outflr) { /* Data header (32bit) Bit 0-3 Reserved Bit 4-7 Compressed type (must be 1 for LZ77) Bit 8-31 Size of decompressed data. if 0, the next 4 bytes are decompressed length Repeat below. Each Flag Byte followed by eight Blocks. Flag data (8bit) Bit 0-7 Type Flags for next 8 Blocks, MSB first Block Type 0 - Uncompressed - Copy 1 Byte from Source to Dest Bit 0-7 One data byte to be copied to dest Block Type 1 - Compressed - Copy LEN Bytes from Dest-Disp-1 to Dest If Reserved is 0: - Default Bit 0-3 Disp MSBs Bit 4-7 LEN - 3 Bit 8-15 Disp LSBs If Reserved is 1: - Higher compression rates for files with (lots of) long repetitions Bit 4-7 Indicator If Indicator > 1: Bit 0-3 Disp MSBs Bit 4-7 LEN - 1 (same bits as Indicator) Bit 8-15 Disp LSBs If Indicator is 1: A(B CD E)(F GH) Bit 0-3 (LEN - 0x111) MSBs Bit 4-7 Indicator; unused Bit 8-15 (LEN- 0x111) 'middle'-SBs Bit 16-19 Disp MSBs Bit 20-23 (LEN - 0x111) LSBs Bit 24-31 Disp LSBs If Indicator is 0: Bit 0-3 (LEN - 0x11) MSBs Bit 4-7 Indicator; unused Bit 8-11 Disp MSBs Bit 12-15 (LEN - 0x11) LSBs Bit 16-23 Disp LSBs */ FileStream fstr = new FileStream(filein, FileMode.Open); if (fstr.Length > int.MaxValue) throw new Exception("Filer larger than 2GB cannot be LZ-0x11-compressed files."); BinaryReader br = new BinaryReader(fstr); int decomp_size = 0, curr_size = 0; int i, j, disp, len; bool flag; byte b1, bt, b2, b3, flags; int cdest; int threshold = 1; if (br.ReadByte() != LZ11_TAG) throw new InvalidDataException(String.Format("File {0:s} is not a valid LZ-0x11 file", filein)); for (i = 0; i < 3; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (1)", filein, MAX_OUTSIZE, decomp_size)); else if (decomp_size == 0) for (i = 0; i < 4; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE << 8) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (2)", filein, MAX_OUTSIZE, decomp_size)); if (showAlways) Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size); byte[] outdata = new byte[decomp_size]; while (curr_size < decomp_size) { try { flags = br.ReadByte(); } catch (EndOfStreamException) { break; } for (i = 0; i < 8 && curr_size < decomp_size; i++) { flag = (flags & (0x80 >> i)) > 0; if (flag) { try { b1 = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } switch (b1 >> 4) { #region case 0 case 0: // ab cd ef // => // len = abc + 0x11 = bc + 0x11 // disp = def len = b1 << 4; try { bt = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } len |= bt >> 4; len += 0x11; disp = (bt & 0x0F) << 8; try { b2 = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } disp |= b2; break; #endregion #region case 1 case 1: // ab cd ef gh // => // len = bcde + 0x111 // disp = fgh // 10 04 92 3F => disp = 0x23F, len = 0x149 + 0x11 = 0x15A try { bt = br.ReadByte(); b2 = br.ReadByte(); b3 = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } len = (b1 & 0xF) << 12; // len = b000 len |= bt << 4; // len = bcd0 len |= (b2 >> 4); // len = bcde len += 0x111; // len = bcde + 0x111 disp = (b2 & 0x0F) << 8; // disp = f disp |= b3; // disp = fgh break; #endregion #region other default: // ab cd // => // len = a + threshold = a + 1 // disp = bcd len = (b1 >> 4) + threshold; disp = (b1 & 0x0F) << 8; try { b2 = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } disp |= b2; break; #endregion } if (disp > curr_size) throw new Exception("Cannot go back more than already written"); cdest = curr_size; for (j = 0; j < len && curr_size < decomp_size; j++) outdata[curr_size++] = outdata[cdest - disp - 1 + j]; if (curr_size > decomp_size) { //throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein)); //Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}.", filein, curr_size, decomp_size)); break; } } else { try { outdata[curr_size++] = br.ReadByte(); } catch (EndOfStreamException) { break; }// throw new Exception("Incomplete data"); } if (curr_size > decomp_size) { //throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein)); //Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}", filein, curr_size, decomp_size)); break; } } } } try { while (br.ReadByte() == 0) { } // if we read a non-zero, print that there is still some data Console.WriteLine("Too much data in file; current INPOS = {0:x}", br.BaseStream.Position - 1); } catch (EndOfStreamException) { } #region save string ext = ""; for (i = 0; i < 4; i++) if (char.IsLetterOrDigit((char)outdata[i])) ext += (char)outdata[i]; else break; if (ext.Length == 0) ext = "dat"; ext = "." + ext; filein = filein.Replace("\\", "/"); outflr = outflr.Replace("\\", "/"); string outfname = filein.Substring(filein.LastIndexOf("/") + 1); if (outfname.Contains(".")) outfname = outfname.Substring(0, outfname.LastIndexOf('.')); if (!outflr.EndsWith("/")) outflr += "/"; while (File.Exists(outflr + outfname + ext)) outfname += "_";/**/ BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.Create)); bw.Write(outdata); bw.Flush(); bw.Close(); #endregion Console.WriteLine("LZ-0x11 Decompressed " + filein); } private static void DecompressFolder(string inflr, string outflr) { showAlways = false; // only print errors/failures if (!outflr.EndsWith("/") && !outflr.EndsWith("\\")) outflr += "/"; StreamWriter sw = null; if (!Directory.Exists(inflr)) { Console.WriteLine("No such file or folder: " + inflr); return; } string[] files = Directory.GetFiles(inflr); foreach (string fname in files) try { Decompress(makeSlashes(fname), outflr); } catch (Exception e) { if (sw == null) sw = new StreamWriter(new FileStream(outflr + "lzsslog.txt", FileMode.Create)); Console.WriteLine(e.Message); sw.WriteLine(e.Message); string copied = fname.Replace(inflr, outflr); if (!File.Exists(copied)) File.Copy(fname, copied); } Console.WriteLine("Done decompressing files in folder " + inflr); if (sw != null) { Console.WriteLine("Errors have been logged to " + outflr + "lzsslog.txt"); sw.Flush(); sw.Close(); } } static void DecompressHuffman(String filename, String outflr) { /* Data Header (32bit) Bit0-3 Data size in bit units (normally 4 or 8) Bit4-7 Compressed type (must be 2 for Huffman) Bit8-31 24bit size of decompressed data in bytes Tree Size (8bit) Bit0-7 Size of Tree Table/2-1 (ie. Offset to Compressed Bitstream) Tree Table (list of 8bit nodes, starting with the root node) Root Node and Non-Data-Child Nodes are: Bit0-5 Offset to next child node, Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2 Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1 Bit6 Node1 End Flag (1=Next child node is data) Bit7 Node0 End Flag (1=Next child node is data) Data nodes are (when End Flag was set in parent node): Bit0-7 Data (upper bits should be zero if Data Size is less than 8) Compressed Bitstream (stored in units of 32bits) Bit0-31 Node Bits (Bit31=First Bit) (0=Node0, 1=Node1) */ BinaryReader br = new BinaryReader(File.OpenRead(filename)); byte firstByte = br.ReadByte(); int dataSize = firstByte & 0x0F; if ((firstByte & 0xF0) != HUFF_TAG) throw new InvalidDataException(String.Format("Invalid huffman comressed file; invalid tag {0:x}", firstByte)); //Console.WriteLine("Data size: {0:x}", dataSize); if (dataSize != 8 && dataSize != 4) throw new InvalidDataException(String.Format("Unhandled dataSize {0:x}", dataSize)); int decomp_size = 0; for (int i = 0; i < 3; i++) { decomp_size |= br.ReadByte() << (i * 8); } //Console.WriteLine("Decompressed size: {0:x}", decomp_size); byte treeSize = br.ReadByte(); HuffTreeNode.maxInpos = 4 + (treeSize + 1) * 2; //Console.WriteLine("Tree Size: {0:x}", treeSize); HuffTreeNode rootNode = new HuffTreeNode(); rootNode.parseData(br); //Console.WriteLine("Tree: {0:s}", rootNode.ToString()); br.BaseStream.Position = 4 + (treeSize + 1) * 2; // go to start of coded bitstream. // read all data uint[] indata = new uint[(br.BaseStream.Length - br.BaseStream.Position) / 4]; for (int i = 0; i < indata.Length; i++) indata[i] = br.ReadUInt32(); //Console.WriteLine(indata[0]); //Console.WriteLine(uint_to_bits(indata[0])); long curr_size = 0; decomp_size *= dataSize == 8 ? 1 : 2; byte[] outdata = new byte[decomp_size]; int idx = -1; string codestr = ""; LinkedList<byte> code = new LinkedList<byte>(); int value; while (curr_size < decomp_size) { try { codestr += uint_to_bits(indata[++idx]); } catch (IndexOutOfRangeException e) { throw new IndexOutOfRangeException("not enough data.", e); } while (codestr.Length > 0) { code.AddFirst(byte.Parse(codestr[0] + "")); codestr = codestr.Remove(0, 1); if (rootNode.getValue(code.Last, out value)) { try { outdata[curr_size++] = (byte)value; } catch (IndexOutOfRangeException ex) { if (code.First.Value != 0) throw ex; } code.Clear(); } } } if (codestr.Length > 0 || idx < indata.Length-1) { while (idx < indata.Length-1) codestr += uint_to_bits(indata[++idx]); codestr = codestr.Replace("0", ""); if (codestr.Length > 0) Console.WriteLine("too much data; str={0:s}, idx={1:g}/{2:g}", codestr, idx, indata.Length); } byte[] realout; if (dataSize == 4) { realout = new byte[decomp_size / 2]; for (int i = 0; i < decomp_size / 2; i++) { if ((outdata[i * 2] & 0xF0) > 0 || (outdata[i * 2 + 1] & 0xF0) > 0) throw new Exception("first 4 bits of data should be 0 if dataSize = 4"); realout[i] = (byte)((outdata[i * 2] << 4) | outdata[i * 2 + 1]); } } else { realout = outdata; } #region save string ext = ""; for (int i = 0; i < 4; i++) if (char.IsLetterOrDigit((char)realout[i])) ext += (char)realout[i]; else break; if (ext.Length == 0) ext = "dat"; ext = "." + ext; filename = filename.Replace("\\", "/"); outflr = outflr.Replace("\\", "/"); string outfname = filename.Substring(filename.LastIndexOf("/") + 1); if (outfname.Contains(".")) outfname = outfname.Substring(0, outfname.LastIndexOf('.')); if (!outflr.EndsWith("/")) outflr += "/"; while (File.Exists(outflr + outfname + ext)) outfname += "_"; BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.CreateNew)); bw.Write(realout); bw.Flush(); bw.Close(); #endregion Console.WriteLine("Huffman decompressed {0:s}", filename); //Console.ReadLine(); /**/ } static void DecompressLZ40(string filein, string outflr) { // no NDSTEK-like specification for this one; I seem to not be able to get those right. /* * byte tag; // 0x40 * byte[3] decompressedSize; * the rest is the data; * * for each chunk: * - first byte determines which blocks are compressed * - block i is compressed iff: * - the i'th MSB is the last 1-bit in the byte * - OR the i'th MSB is a 0-bit, not directly followed by other 0-bits. * - note that there will never be more than one 0-bit before any 1-bit in this byte * (look at the corresponding code, it may clarify this a bit more) * - then come 8 blocks: * - a non-compressed block is simply one single byte * - a compressed block can have 3 sizes: * - A0 CD EF * -> Length = EF + 0x10, Disp = CDA * - A1 CD EF GH * -> Length = GHEF + 0x110, Disp = CDA * - AB CD (B > 1) * -> Length = B, Disp = CDA * Copy <Length> bytes from Dest-<Disp> to Dest (with <Dest> similar to the NDSTEK specs) */ FileStream fstr = new FileStream(filein, FileMode.Open); if (fstr.Length > int.MaxValue) throw new Exception("Filer larger than 2GB cannot be LZSS-compressed files."); BinaryReader br = new BinaryReader(fstr); int decomp_size = 0, curr_size = 0; if (br.ReadByte() != LZ40_TAG) throw new InvalidDataException(String.Format("File {0:s} is not a valid LZSS-11 file", filein)); for (int i = 0; i < 3; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (1)", filein, MAX_OUTSIZE, decomp_size)); else if (decomp_size == 0) for (int i = 0; i < 4; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE << 8) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (2)", filein, MAX_OUTSIZE, decomp_size)); if (showAlways) Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size); byte[] outdata = new byte[decomp_size]; while (curr_size < decomp_size) { int flag; try { flag = br.ReadByte(); } catch (EndOfStreamException) { Console.WriteLine("Not enough data"); break; } int flagB = flag; bool[] compFlags = new bool[8]; bool[] fbits = new bool[11]; fbits[0] = true; fbits[9] = false; fbits[10] = false; // determine which blocks are compressed int b = 0; while (flag > 0) { bool bit = (flag & 0x80) > 0; flag = (flag & 0x7F) << 1; compFlags[b++] = (flag == 0) || !bit; } /* Console.WriteLine("Flag: 0x{0:X2}", flagB); Console.Write("-> ( "); for (int i = 0; i < 8; i++) Console.Write(compFlags[i] ? "1," : "0,"); Console.WriteLine(")");/**/ for (int i = 0; i < 8 && curr_size < decomp_size; i++) { if (compFlags[i]) { ushort compressed = br.ReadUInt16(); // ABCD (or CD AB if read byte-by-byte) // -> D is length // -> ABC is disp int len = compressed & 0x000F; int disp = compressed >> 4; // if D == 0, actual format is: // CD AB EF // -> DEF is length - 0x10 // -> ABC is disp // if D == 1, actual format is: // CD AB EF GH // -> GHEF is length - 0x110 // -> ABC is disp if (len == 0) len = br.ReadByte() + 0x10; else if (len == 1) len = br.ReadUInt16() + 0x110; if (disp > curr_size) throw new Exception("Cannot go back more than already written " + "(compressed block=0x" + compressed.ToString("X4") + ")\n" + "INPOS = 0x" + (br.BaseStream.Position - 2).ToString("X4")); for (int j = 0; j < len; j++) { outdata[curr_size + j] = outdata[curr_size - disp + j]; } curr_size += len; } else { outdata[curr_size++] = br.ReadByte(); } } } try { byte b; while ((b = br.ReadByte()) == 0 || b == 0x80) { } // if we read a non-zero up to the end of the file, print that there is still some data // (0x40 compression seems to add 80 00 00 sometimes, so also ignore 0x80-bytes) Console.WriteLine("Too much data in file; current INPOS = {0:x}", br.BaseStream.Position - 1); } catch (EndOfStreamException) { } #region save string ext = ""; for (int i = 0; i < 4; i++) if (char.IsLetterOrDigit((char)outdata[i])) ext += (char)outdata[i]; else break; if (ext.Length == 0) ext = "dat"; ext = "." + ext; filein = filein.Replace("\\", "/"); outflr = outflr.Replace("\\", "/"); string outfname = filein.Substring(filein.LastIndexOf("/") + 1); if (outfname.Contains(".")) outfname = outfname.Substring(0, outfname.LastIndexOf('.')); if (!outflr.EndsWith("/")) outflr += "/"; while (File.Exists(outflr + outfname + ext)) outfname += "_";/**/ BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.Create)); bw.Write(outdata); bw.Flush(); bw.Close(); #endregion Console.WriteLine("LZ-0x40-decompressed " + filein); } static void DecompressLZ77(string filein, string outflr) { /* Data header (32bit) Bit 0-3 Reserved Bit 4-7 Compressed type (must be 1 for LZ77) Bit 8-31 Size of decompressed data Repeat below. Each Flag Byte followed by eight Blocks. Flag data (8bit) Bit 0-7 Type Flags for next 8 Blocks, MSB first Block Type 0 - Uncompressed - Copy 1 Byte from Source to Dest Bit 0-7 One data byte to be copied to dest Block Type 1 - Compressed - Copy N+3 Bytes from Dest-Disp-1 to Dest Bit 0-3 Disp MSBs Bit 4-7 Number of bytes to copy (minus 3) Bit 8-15 Disp LSBs */ FileStream fstr = new FileStream(filein, FileMode.Open); if (fstr.Length > int.MaxValue) throw new Exception("Filer larger than 2GB cannot be LZ-0x10-compressed files."); BinaryReader br = new BinaryReader(fstr); long decomp_size = 0, curr_size = 0; int flags, i, j, disp, n; bool flag; byte b; long cdest; if (br.ReadByte() != LZ10_TAG) throw new InvalidDataException(String.Format("File {0:s} is not a valid LZ-0x10 file", filein)); for (i = 0; i < 3; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed.", filein, MAX_OUTSIZE, decomp_size)); else if (decomp_size == 0) for (i = 0; i < 4; i++) decomp_size += br.ReadByte() << (i * 8); if (decomp_size > MAX_OUTSIZE << 8) throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed.", filein, MAX_OUTSIZE, decomp_size)); if (showAlways) Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size); #region decompress byte[] outdata = new byte[decomp_size]; while (curr_size < decomp_size) { try { flags = br.ReadByte(); } catch (EndOfStreamException) { break; } for (i = 0; i < 8; i++) { flag = (flags & (0x80 >> i)) > 0; if (flag) { disp = 0; try { b = br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } n = b >> 4; disp = (b & 0x0F) << 8; try { disp |= br.ReadByte(); } catch (EndOfStreamException) { throw new Exception("Incomplete data"); } n += 3; cdest = curr_size; //Console.WriteLine("disp: 0x{0:x}", disp); if (disp > curr_size) throw new Exception("Cannot go back more than already written"); for (j = 0; j < n; j++) outdata[curr_size++] = outdata[cdest - disp - 1 + j]; //curr_size += len; if (curr_size > decomp_size) { //throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein)); //Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}.", filein, curr_size, decomp_size)); break; } } else { try { b = br.ReadByte(); } catch (EndOfStreamException) { break;}// throw new Exception("Incomplete data"); } try { outdata[curr_size++] = b; } catch (IndexOutOfRangeException) { if (b == 0) break; } //curr_size++; if (curr_size > decomp_size) { //throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein)); //Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}", filein, curr_size, decomp_size)); break; } } } } try { while (br.ReadByte() == 0) { } // if we read a non-zero, print that there is still some data Console.WriteLine("Too many data in file; current INPOS = {0:x}", br.BaseStream.Position - 1); } catch (EndOfStreamException) { } #endregion #region save string ext = ""; for (i = 0; i < 4; i++) if (char.IsLetterOrDigit((char)outdata[i])) ext += (char)outdata[i]; else break; if (ext.Length == 0) ext = "dat"; ext = "." + ext; filein = filein.Replace("\\", "/"); outflr = outflr.Replace("\\", "/"); string outfname = filein.Substring(filein.LastIndexOf("/") + 1); if (outfname.Contains(".")) outfname = outfname.Substring(0, outfname.LastIndexOf('.')); if (!outflr.EndsWith("/")) outflr += "/"; while (File.Exists(outflr + outfname + ext)) outfname += "_"; BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.CreateNew)); bw.Write(outdata); bw.Flush(); bw.Close(); #endregion Console.WriteLine("LZ-0x10 Decompressed " + filein); }
internal void parseData(BinaryReader br) { /* * Tree Table (list of 8bit nodes, starting with the root node) Root Node and Non-Data-Child Nodes are: Bit0-5 Offset to next child node, Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2 Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1 Bit6 Node1 End Flag (1=Next child node is data) Bit7 Node0 End Flag (1=Next child node is data) Data nodes are (when End Flag was set in parent node): Bit0-7 Data (upper bits should be zero if Data Size is less than 8) */ this.node0 = new HuffTreeNode(); this.node1 = new HuffTreeNode(); long currPos = br.BaseStream.Position; byte b = br.ReadByte(); long offset = b & 0x3F; bool end0 = (b & 0x80) > 0, end1 = (b & 0x40) > 0; // parse data for node0 br.BaseStream.Position = (currPos - (currPos & 1)) + offset * 2 + 2; if (br.BaseStream.Position < maxInpos) { if (end0) node0.data = br.ReadByte(); else node0.parseData(br); } // parse data for node1 br.BaseStream.Position = (currPos - (currPos & 1)) + offset * 2 + 2 + 1; if (br.BaseStream.Position < maxInpos) { if (end1) node1.data = br.ReadByte(); else node1.parseData(br); } // reset position br.BaseStream.Position = currPos; }