static void Decompress11LZS(string filein, string outflr)
{
/* Data header (32bit)
Bit 0-3 Reserved
Bit 4-7 Compressed type (must be 1 for LZ77)
Bit 8-31 Size of decompressed data. if 0, the next 4 bytes are decompressed length
Repeat below. Each Flag Byte followed by eight Blocks.
Flag data (8bit)
Bit 0-7 Type Flags for next 8 Blocks, MSB first
Block Type 0 - Uncompressed - Copy 1 Byte from Source to Dest
Bit 0-7 One data byte to be copied to dest
Block Type 1 - Compressed - Copy LEN Bytes from Dest-Disp-1 to Dest
If Reserved is 0: - Default
Bit 0-3 Disp MSBs
Bit 4-7 LEN - 3
Bit 8-15 Disp LSBs
If Reserved is 1: - Higher compression rates for files with (lots of) long repetitions
Bit 4-7 Indicator
If Indicator > 1:
Bit 0-3 Disp MSBs
Bit 4-7 LEN - 1 (same bits as Indicator)
Bit 8-15 Disp LSBs
If Indicator is 1: A(B CD E)(F GH)
Bit 0-3 (LEN - 0x111) MSBs
Bit 4-7 Indicator; unused
Bit 8-15 (LEN- 0x111) 'middle'-SBs
Bit 16-19 Disp MSBs
Bit 20-23 (LEN - 0x111) LSBs
Bit 24-31 Disp LSBs
If Indicator is 0:
Bit 0-3 (LEN - 0x11) MSBs
Bit 4-7 Indicator; unused
Bit 8-11 Disp MSBs
Bit 12-15 (LEN - 0x11) LSBs
Bit 16-23 Disp LSBs
*/
FileStream fstr = new FileStream(filein, FileMode.Open);
if (fstr.Length > int.MaxValue)
throw new Exception("Filer larger than 2GB cannot be LZ-0x11-compressed files.");
BinaryReader br = new BinaryReader(fstr);
int decomp_size = 0, curr_size = 0;
int i, j, disp, len;
bool flag;
byte b1, bt, b2, b3, flags;
int cdest;
int threshold = 1;
if (br.ReadByte() != LZ11_TAG)
throw new InvalidDataException(String.Format("File {0:s} is not a valid LZ-0x11 file", filein));
for (i = 0; i < 3; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (1)", filein, MAX_OUTSIZE, decomp_size));
else if (decomp_size == 0)
for (i = 0; i < 4; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE << 8)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (2)", filein, MAX_OUTSIZE, decomp_size));
if (showAlways)
Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size);
byte[] outdata = new byte[decomp_size];
while (curr_size < decomp_size)
{
try { flags = br.ReadByte(); }
catch (EndOfStreamException) { break; }
for (i = 0; i < 8 && curr_size < decomp_size; i++)
{
flag = (flags & (0x80 >> i)) > 0;
if (flag)
{
try { b1 = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
switch (b1 >> 4)
{
#region case 0
case 0:
// ab cd ef
// =>
// len = abc + 0x11 = bc + 0x11
// disp = def
len = b1 << 4;
try { bt = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
len |= bt >> 4;
len += 0x11;
disp = (bt & 0x0F) << 8;
try { b2 = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
disp |= b2;
break;
#endregion
#region case 1
case 1:
// ab cd ef gh
// =>
// len = bcde + 0x111
// disp = fgh
// 10 04 92 3F => disp = 0x23F, len = 0x149 + 0x11 = 0x15A
try { bt = br.ReadByte(); b2 = br.ReadByte(); b3 = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
len = (b1 & 0xF) << 12; // len = b000
len |= bt << 4; // len = bcd0
len |= (b2 >> 4); // len = bcde
len += 0x111; // len = bcde + 0x111
disp = (b2 & 0x0F) << 8; // disp = f
disp |= b3; // disp = fgh
break;
#endregion
#region other
default:
// ab cd
// =>
// len = a + threshold = a + 1
// disp = bcd
len = (b1 >> 4) + threshold;
disp = (b1 & 0x0F) << 8;
try { b2 = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
disp |= b2;
break;
#endregion
}
if (disp > curr_size)
throw new Exception("Cannot go back more than already written");
cdest = curr_size;
for (j = 0; j < len && curr_size < decomp_size; j++)
outdata[curr_size++] = outdata[cdest - disp - 1 + j];
if (curr_size > decomp_size)
{
//throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein));
//Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}.", filein, curr_size, decomp_size));
break;
}
}
else
{
try { outdata[curr_size++] = br.ReadByte(); }
catch (EndOfStreamException) { break; }// throw new Exception("Incomplete data"); }
if (curr_size > decomp_size)
{
//throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein));
//Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}", filein, curr_size, decomp_size));
break;
}
}
}
}
try
{
while (br.ReadByte() == 0) { } // if we read a non-zero, print that there is still some data
Console.WriteLine("Too much data in file; current INPOS = {0:x}", br.BaseStream.Position - 1);
}
catch (EndOfStreamException) { }
#region save
string ext = "";
for (i = 0; i < 4; i++)
if (char.IsLetterOrDigit((char)outdata[i]))
ext += (char)outdata[i];
else
break;
if (ext.Length == 0)
ext = "dat";
ext = "." + ext;
filein = filein.Replace("\\", "/");
outflr = outflr.Replace("\\", "/");
string outfname = filein.Substring(filein.LastIndexOf("/") + 1);
if (outfname.Contains("."))
outfname = outfname.Substring(0, outfname.LastIndexOf('.'));
if (!outflr.EndsWith("/"))
outflr += "/";
while (File.Exists(outflr + outfname + ext))
outfname += "_";/**/
BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.Create));
bw.Write(outdata);
bw.Flush();
bw.Close();
#endregion
Console.WriteLine("LZ-0x11 Decompressed " + filein);
}
private static void DecompressFolder(string inflr, string outflr)
{
showAlways = false; // only print errors/failures
if (!outflr.EndsWith("/") && !outflr.EndsWith("\\"))
outflr += "/";
StreamWriter sw = null;
if (!Directory.Exists(inflr))
{
Console.WriteLine("No such file or folder: " + inflr);
return;
}
string[] files = Directory.GetFiles(inflr);
foreach (string fname in files)
try
{
Decompress(makeSlashes(fname), outflr);
}
catch (Exception e)
{
if (sw == null)
sw = new StreamWriter(new FileStream(outflr + "lzsslog.txt", FileMode.Create));
Console.WriteLine(e.Message);
sw.WriteLine(e.Message);
string copied = fname.Replace(inflr, outflr);
if (!File.Exists(copied))
File.Copy(fname, copied);
}
Console.WriteLine("Done decompressing files in folder " + inflr);
if (sw != null)
{
Console.WriteLine("Errors have been logged to " + outflr + "lzsslog.txt");
sw.Flush();
sw.Close();
}
}
static void DecompressHuffman(String filename, String outflr)
{
/*
Data Header (32bit)
Bit0-3 Data size in bit units (normally 4 or 8)
Bit4-7 Compressed type (must be 2 for Huffman)
Bit8-31 24bit size of decompressed data in bytes
Tree Size (8bit)
Bit0-7 Size of Tree Table/2-1 (ie. Offset to Compressed Bitstream)
Tree Table (list of 8bit nodes, starting with the root node)
Root Node and Non-Data-Child Nodes are:
Bit0-5 Offset to next child node,
Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2
Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1
Bit6 Node1 End Flag (1=Next child node is data)
Bit7 Node0 End Flag (1=Next child node is data)
Data nodes are (when End Flag was set in parent node):
Bit0-7 Data (upper bits should be zero if Data Size is less than 8)
Compressed Bitstream (stored in units of 32bits)
Bit0-31 Node Bits (Bit31=First Bit) (0=Node0, 1=Node1)
*/
BinaryReader br = new BinaryReader(File.OpenRead(filename));
byte firstByte = br.ReadByte();
int dataSize = firstByte & 0x0F;
if ((firstByte & 0xF0) != HUFF_TAG)
throw new InvalidDataException(String.Format("Invalid huffman comressed file; invalid tag {0:x}", firstByte));
//Console.WriteLine("Data size: {0:x}", dataSize);
if (dataSize != 8 && dataSize != 4)
throw new InvalidDataException(String.Format("Unhandled dataSize {0:x}", dataSize));
int decomp_size = 0;
for (int i = 0; i < 3; i++)
{
decomp_size |= br.ReadByte() << (i * 8);
}
//Console.WriteLine("Decompressed size: {0:x}", decomp_size);
byte treeSize = br.ReadByte();
HuffTreeNode.maxInpos = 4 + (treeSize + 1) * 2;
//Console.WriteLine("Tree Size: {0:x}", treeSize);
HuffTreeNode rootNode = new HuffTreeNode();
rootNode.parseData(br);
//Console.WriteLine("Tree: {0:s}", rootNode.ToString());
br.BaseStream.Position = 4 + (treeSize + 1) * 2; // go to start of coded bitstream.
// read all data
uint[] indata = new uint[(br.BaseStream.Length - br.BaseStream.Position) / 4];
for (int i = 0; i < indata.Length; i++)
indata[i] = br.ReadUInt32();
//Console.WriteLine(indata[0]);
//Console.WriteLine(uint_to_bits(indata[0]));
long curr_size = 0;
decomp_size *= dataSize == 8 ? 1 : 2;
byte[] outdata = new byte[decomp_size];
int idx = -1;
string codestr = "";
LinkedList<byte> code = new LinkedList<byte>();
int value;
while (curr_size < decomp_size)
{
try
{
codestr += uint_to_bits(indata[++idx]);
}
catch (IndexOutOfRangeException e)
{
throw new IndexOutOfRangeException("not enough data.", e);
}
while (codestr.Length > 0)
{
code.AddFirst(byte.Parse(codestr[0] + ""));
codestr = codestr.Remove(0, 1);
if (rootNode.getValue(code.Last, out value))
{
try
{
outdata[curr_size++] = (byte)value;
}
catch (IndexOutOfRangeException ex)
{
if (code.First.Value != 0)
throw ex;
}
code.Clear();
}
}
}
if (codestr.Length > 0 || idx < indata.Length-1)
{
while (idx < indata.Length-1)
codestr += uint_to_bits(indata[++idx]);
codestr = codestr.Replace("0", "");
if (codestr.Length > 0)
Console.WriteLine("too much data; str={0:s}, idx={1:g}/{2:g}", codestr, idx, indata.Length);
}
byte[] realout;
if (dataSize == 4)
{
realout = new byte[decomp_size / 2];
for (int i = 0; i < decomp_size / 2; i++)
{
if ((outdata[i * 2] & 0xF0) > 0
|| (outdata[i * 2 + 1] & 0xF0) > 0)
throw new Exception("first 4 bits of data should be 0 if dataSize = 4");
realout[i] = (byte)((outdata[i * 2] << 4) | outdata[i * 2 + 1]);
}
}
else
{
realout = outdata;
}
#region save
string ext = "";
for (int i = 0; i < 4; i++)
if (char.IsLetterOrDigit((char)realout[i]))
ext += (char)realout[i];
else
break;
if (ext.Length == 0)
ext = "dat";
ext = "." + ext;
filename = filename.Replace("\\", "/");
outflr = outflr.Replace("\\", "/");
string outfname = filename.Substring(filename.LastIndexOf("/") + 1);
if (outfname.Contains("."))
outfname = outfname.Substring(0, outfname.LastIndexOf('.'));
if (!outflr.EndsWith("/"))
outflr += "/";
while (File.Exists(outflr + outfname + ext))
outfname += "_";
BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.CreateNew));
bw.Write(realout);
bw.Flush();
bw.Close();
#endregion
Console.WriteLine("Huffman decompressed {0:s}", filename);
//Console.ReadLine();
/**/
}
static void DecompressLZ40(string filein, string outflr)
{
// no NDSTEK-like specification for this one; I seem to not be able to get those right.
/*
* byte tag; // 0x40
* byte[3] decompressedSize;
* the rest is the data;
*
* for each chunk:
* - first byte determines which blocks are compressed
* - block i is compressed iff:
* - the i'th MSB is the last 1-bit in the byte
* - OR the i'th MSB is a 0-bit, not directly followed by other 0-bits.
* - note that there will never be more than one 0-bit before any 1-bit in this byte
* (look at the corresponding code, it may clarify this a bit more)
* - then come 8 blocks:
* - a non-compressed block is simply one single byte
* - a compressed block can have 3 sizes:
* - A0 CD EF
* -> Length = EF + 0x10, Disp = CDA
* - A1 CD EF GH
* -> Length = GHEF + 0x110, Disp = CDA
* - AB CD (B > 1)
* -> Length = B, Disp = CDA
* Copy <Length> bytes from Dest-<Disp> to Dest (with <Dest> similar to the NDSTEK specs)
*/
FileStream fstr = new FileStream(filein, FileMode.Open);
if (fstr.Length > int.MaxValue)
throw new Exception("Filer larger than 2GB cannot be LZSS-compressed files.");
BinaryReader br = new BinaryReader(fstr);
int decomp_size = 0, curr_size = 0;
if (br.ReadByte() != LZ40_TAG)
throw new InvalidDataException(String.Format("File {0:s} is not a valid LZSS-11 file", filein));
for (int i = 0; i < 3; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (1)", filein, MAX_OUTSIZE, decomp_size));
else if (decomp_size == 0)
for (int i = 0; i < 4; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE << 8)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed. (2)", filein, MAX_OUTSIZE, decomp_size));
if (showAlways)
Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size);
byte[] outdata = new byte[decomp_size];
while (curr_size < decomp_size)
{
int flag;
try { flag = br.ReadByte(); }
catch (EndOfStreamException)
{
Console.WriteLine("Not enough data");
break;
}
int flagB = flag;
bool[] compFlags = new bool[8];
bool[] fbits = new bool[11];
fbits[0] = true;
fbits[9] = false;
fbits[10] = false;
// determine which blocks are compressed
int b = 0;
while (flag > 0)
{
bool bit = (flag & 0x80) > 0;
flag = (flag & 0x7F) << 1;
compFlags[b++] = (flag == 0) || !bit;
}
/*
Console.WriteLine("Flag: 0x{0:X2}", flagB);
Console.Write("-> ( ");
for (int i = 0; i < 8; i++)
Console.Write(compFlags[i] ? "1," : "0,");
Console.WriteLine(")");/**/
for (int i = 0; i < 8 && curr_size < decomp_size; i++)
{
if (compFlags[i])
{
ushort compressed = br.ReadUInt16();
// ABCD (or CD AB if read byte-by-byte)
// -> D is length
// -> ABC is disp
int len = compressed & 0x000F;
int disp = compressed >> 4;
// if D == 0, actual format is:
// CD AB EF
// -> DEF is length - 0x10
// -> ABC is disp
// if D == 1, actual format is:
// CD AB EF GH
// -> GHEF is length - 0x110
// -> ABC is disp
if (len == 0)
len = br.ReadByte() + 0x10;
else if (len == 1)
len = br.ReadUInt16() + 0x110;
if (disp > curr_size)
throw new Exception("Cannot go back more than already written "
+ "(compressed block=0x" + compressed.ToString("X4") + ")\n"
+ "INPOS = 0x" + (br.BaseStream.Position - 2).ToString("X4"));
for (int j = 0; j < len; j++)
{
outdata[curr_size + j] = outdata[curr_size - disp + j];
}
curr_size += len;
}
else
{
outdata[curr_size++] = br.ReadByte();
}
}
}
try
{
byte b;
while ((b = br.ReadByte()) == 0
|| b == 0x80) { }
// if we read a non-zero up to the end of the file, print that there is still some data
// (0x40 compression seems to add 80 00 00 sometimes, so also ignore 0x80-bytes)
Console.WriteLine("Too much data in file; current INPOS = {0:x}", br.BaseStream.Position - 1);
}
catch (EndOfStreamException) { }
#region save
string ext = "";
for (int i = 0; i < 4; i++)
if (char.IsLetterOrDigit((char)outdata[i]))
ext += (char)outdata[i];
else
break;
if (ext.Length == 0)
ext = "dat";
ext = "." + ext;
filein = filein.Replace("\\", "/");
outflr = outflr.Replace("\\", "/");
string outfname = filein.Substring(filein.LastIndexOf("/") + 1);
if (outfname.Contains("."))
outfname = outfname.Substring(0, outfname.LastIndexOf('.'));
if (!outflr.EndsWith("/"))
outflr += "/";
while (File.Exists(outflr + outfname + ext))
outfname += "_";/**/
BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.Create));
bw.Write(outdata);
bw.Flush();
bw.Close();
#endregion
Console.WriteLine("LZ-0x40-decompressed " + filein);
}
static void DecompressLZ77(string filein, string outflr)
{
/* Data header (32bit)
Bit 0-3 Reserved
Bit 4-7 Compressed type (must be 1 for LZ77)
Bit 8-31 Size of decompressed data
Repeat below. Each Flag Byte followed by eight Blocks.
Flag data (8bit)
Bit 0-7 Type Flags for next 8 Blocks, MSB first
Block Type 0 - Uncompressed - Copy 1 Byte from Source to Dest
Bit 0-7 One data byte to be copied to dest
Block Type 1 - Compressed - Copy N+3 Bytes from Dest-Disp-1 to Dest
Bit 0-3 Disp MSBs
Bit 4-7 Number of bytes to copy (minus 3)
Bit 8-15 Disp LSBs
*/
FileStream fstr = new FileStream(filein, FileMode.Open);
if (fstr.Length > int.MaxValue)
throw new Exception("Filer larger than 2GB cannot be LZ-0x10-compressed files.");
BinaryReader br = new BinaryReader(fstr);
long decomp_size = 0, curr_size = 0;
int flags, i, j, disp, n;
bool flag;
byte b;
long cdest;
if (br.ReadByte() != LZ10_TAG)
throw new InvalidDataException(String.Format("File {0:s} is not a valid LZ-0x10 file", filein));
for (i = 0; i < 3; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed.", filein, MAX_OUTSIZE, decomp_size));
else if (decomp_size == 0)
for (i = 0; i < 4; i++)
decomp_size += br.ReadByte() << (i * 8);
if (decomp_size > MAX_OUTSIZE << 8)
throw new Exception(String.Format("{0:s} will be larger than 0x{1:x} (0x{2:x}) and will not be decompressed.", filein, MAX_OUTSIZE, decomp_size));
if (showAlways)
Console.WriteLine("Decompressing {0:s}. (outsize: 0x{1:x})", filein, decomp_size);
#region decompress
byte[] outdata = new byte[decomp_size];
while (curr_size < decomp_size)
{
try { flags = br.ReadByte(); }
catch (EndOfStreamException) { break; }
for (i = 0; i < 8; i++)
{
flag = (flags & (0x80 >> i)) > 0;
if (flag)
{
disp = 0;
try { b = br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
n = b >> 4;
disp = (b & 0x0F) << 8;
try { disp |= br.ReadByte(); }
catch (EndOfStreamException) { throw new Exception("Incomplete data"); }
n += 3;
cdest = curr_size;
//Console.WriteLine("disp: 0x{0:x}", disp);
if (disp > curr_size)
throw new Exception("Cannot go back more than already written");
for (j = 0; j < n; j++)
outdata[curr_size++] = outdata[cdest - disp - 1 + j];
//curr_size += len;
if (curr_size > decomp_size)
{
//throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein));
//Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}.", filein, curr_size, decomp_size));
break;
}
}
else
{
try { b = br.ReadByte(); }
catch (EndOfStreamException) { break;}// throw new Exception("Incomplete data"); }
try { outdata[curr_size++] = b; }
catch (IndexOutOfRangeException) { if (b == 0) break; }
//curr_size++;
if (curr_size > decomp_size)
{
//throw new Exception(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header", filein));
//Console.WriteLine(String.Format("File {0:s} is not a valid LZ77 file; actual output size > output size in header; {1:x} > {2:x}", filein, curr_size, decomp_size));
break;
}
}
}
}
try
{
while (br.ReadByte() == 0) { } // if we read a non-zero, print that there is still some data
Console.WriteLine("Too many data in file; current INPOS = {0:x}", br.BaseStream.Position - 1);
}
catch (EndOfStreamException) { }
#endregion
#region save
string ext = "";
for (i = 0; i < 4; i++)
if (char.IsLetterOrDigit((char)outdata[i]))
ext += (char)outdata[i];
else
break;
if (ext.Length == 0)
ext = "dat";
ext = "." + ext;
filein = filein.Replace("\\", "/");
outflr = outflr.Replace("\\", "/");
string outfname = filein.Substring(filein.LastIndexOf("/") + 1);
if (outfname.Contains("."))
outfname = outfname.Substring(0, outfname.LastIndexOf('.'));
if (!outflr.EndsWith("/"))
outflr += "/";
while (File.Exists(outflr + outfname + ext))
outfname += "_";
BinaryWriter bw = new BinaryWriter(new FileStream(outflr + outfname + ext, FileMode.CreateNew));
bw.Write(outdata);
bw.Flush();
bw.Close();
#endregion
Console.WriteLine("LZ-0x10 Decompressed " + filein);
}