/// <summary>
/// Gets the optimal compression lengths for each start of a compressed block using Dynamic Programming.
/// This takes O(n^2) time.
/// </summary>
/// <param name="indata">The data to compress.</param>
/// <param name="inLength">The length of the data to compress.</param>
/// <param name="lengths">The optimal 'length' of the compressed blocks. For each byte in the input data,
/// this value is the optimal 'length' value. If it is 1, the block should not be compressed.</param>
/// <param name="disps">The 'disp' values of the compressed blocks. May be less than 3, in which case the
/// corresponding length will never be anything other than 1.</param>
private unsafe void GetOptimalCompressionLengths(byte *indata, int inLength, out int[] lengths, out int[] disps)
{
lengths = new int[inLength];
disps = new int[inLength];
int[] minLengths = new int[inLength];
for (int i = inLength - 1; i >= 0; i--)
{
// first get the compression length when the next byte is not compressed
minLengths[i] = int.MaxValue;
lengths[i] = 1;
if (i + 1 >= inLength)
{
minLengths[i] = 1;
}
else
{
minLengths[i] = 1 + minLengths[i + 1];
}
// then the optimal compressed length
int oldLength = Math.Min(0x1001, i);
// get the appropriate disp while at it. Takes at most O(n) time if oldLength is considered O(n)
// be sure to bound the input length with 0x12, as that's the maximum length for LZ-Ovl compressed blocks.
int maxLen = LZUtil.GetOccurrenceLength(indata + i, Math.Min(inLength - i, 0x12),
indata + i - oldLength, oldLength, out disps[i]);
if (disps[i] > i)
{
throw new Exception("disp is too large");
}
// disp < 3 cannot be stored explicitly.
if (disps[i] < 3)
{
maxLen = 1;
}
for (int j = 3; j <= maxLen; j++)
{
int newCompLen;
if (i + j >= inLength)
{
newCompLen = 2;
}
else
{
newCompLen = 2 + minLengths[i + j];
}
if (newCompLen < minLengths[i])
{
lengths[i] = j;
minLengths[i] = newCompLen;
}
}
}
// we could optimize this further to also optimize it with regard to the flag-bytes, but that would require 8 times
// more space and time (one for each position in the block) for only a potentially tiny increase in compression ratio.
}
private unsafe void GetOptimalCompressionLengths(byte *indata, int inLength, out int[] lengths, out int[] disps)
{
lengths = new int[inLength];
disps = new int[inLength];
int[] numArray = new int[inLength];
for (int i = inLength - 1; i >= 0; i--)
{
numArray[i] = 0x7fffffff;
lengths[i] = 1;
if ((i + 1) >= inLength)
{
numArray[i] = 1;
}
else
{
numArray[i] = 1 + numArray[i + 1];
}
int oldLength = Math.Min(0x1000, i);
int num3 = LZUtil.GetOccurrenceLength(indata + i, Math.Min(inLength - i, 0x10110), (indata + i) - oldLength, oldLength, out disps[i]);
for (int j = 3; j <= num3; j++)
{
int num5;
int num6;
if (j > 0x110)
{
num5 = 4;
}
else if (j > 0x10)
{
num5 = 3;
}
else
{
num5 = 2;
}
if ((i + j) >= inLength)
{
num6 = num5;
}
else
{
num6 = num5 + numArray[i + j];
}
if (num6 < numArray[i])
{
lengths[i] = j;
numArray[i] = num6;
}
}
}
}
/// <summary>
/// Gets the optimal compression lengths for each start of a compressed block using Dynamic Programming.
/// This takes O(n^2) time, although in practice it will often be O(n^3) since one of the constants is 0x10110
/// (the maximum length of a compressed block)
/// </summary>
/// <param name="indata">The data to compress.</param>
/// <param name="inLength">The length of the data to compress.</param>
/// <param name="lengths">The optimal 'length' of the compressed blocks. For each byte in the input data,
/// this value is the optimal 'length' value. If it is 1, the block should not be compressed.</param>
/// <param name="disps">The 'disp' values of the compressed blocks. May be 0, in which case the
/// corresponding length will never be anything other than 1.</param>
private unsafe void GetOptimalCompressionLengths(byte* indata, int inLength, out int[] lengths, out int[] disps)
{
lengths = new int[inLength];
disps = new int[inLength];
int[] minLengths = new int[inLength];
for (int i = inLength - 1; i >= 0; i--)
{
// first get the compression length when the next byte is not compressed
minLengths[i] = int.MaxValue;
lengths[i] = 1;
if (i + 1 >= inLength)
minLengths[i] = 1;
else
minLengths[i] = 1 + minLengths[i + 1];
// then the optimal compressed length
int oldLength = Math.Min(0x1000, i);
// get the appropriate disp while at it. Takes at most O(n) time if oldLength is considered O(n) and 0x10110 constant.
// however since a lot of files will not be larger than 0x10110, this will often take ~O(n^2) time.
// be sure to bound the input length with 0x10110, as that's the maximum length for LZ-11 compressed blocks.
int maxLen = LZUtil.GetOccurrenceLength(indata + i, Math.Min(inLength - i, 0x10110),
indata + i - oldLength, oldLength, out disps[i]);
if (disps[i] > i)
throw new Exception(Main.Get_Traduction("S02"));
for (int j = 3; j <= maxLen; j++)
{
int blocklen;
if (j > 0x110)
blocklen = 4;
else if (j > 0x10)
blocklen = 3;
else
blocklen = 2;
int newCompLen;
if (i + j >= inLength)
newCompLen = blocklen;
else
newCompLen = blocklen + minLengths[i + j];
if (newCompLen < minLengths[i])
{
lengths[i] = j;
minLengths[i] = newCompLen;
}
}
}
// we could optimize this further to also optimize it with regard to the flag-bytes, but that would require 8 times
// more space and time (one for each position in the block) for only a potentially tiny increase in compression ratio.
}
public override unsafe int Compress(Stream instream, long inLength, Stream outstream)
{
if (lookAhead)
{
return(CompressWithLA(instream, inLength, outstream));
}
byte[] buffer = new byte[inLength];
int num = instream.Read(buffer, 0, (int)inLength);
outstream.WriteByte(base.magicByte);
outstream.WriteByte((byte)(inLength & 0xffL));
outstream.WriteByte((byte)((inLength >> 8) & 0xffL));
outstream.WriteByte((byte)((inLength >> 0x10) & 0xffL));
int num2 = 4;
fixed(byte *numRef = buffer)
{
byte[] buffer2 = new byte[0x21];
buffer2[0] = 0;
int count = 1;
int num4 = 0;
int num5 = 0;
while (num5 < inLength)
{
int num6;
if (num4 == 8)
{
outstream.Write(buffer2, 0, count);
num2 += count;
buffer2[0] = 0;
count = 1;
num4 = 0;
}
int oldLength = Math.Min(num5, 0x1000);
int num8 = LZUtil.GetOccurrenceLength(numRef + num5, (int)Math.Min((long)(inLength - num5), (long)0x10110L), (numRef + num5) - oldLength, oldLength, out num6);
if (num8 < 3)
{
buffer2[count++] = numRef[num5++];
}
else
{
num5 += num8;
buffer2[0] = (byte)(buffer2[0] | ((byte)(((int)1) << (7 - num4))));
if (num8 > 0x110)
{
buffer2[count] = 0x10;
buffer2[count] = (byte)(buffer2[count] | ((byte)(((num8 - 0x111) >> 12) & 15)));
count++;
buffer2[count] = (byte)(((num8 - 0x111) >> 4) & 0xff);
count++;
buffer2[count] = (byte)(((num8 - 0x111) << 4) & 240);
}
else if (num8 > 0x10)
{
buffer2[count] = 0;
buffer2[count] = (byte)(buffer2[count] | ((byte)(((num8 - 0x111) >> 4) & 15)));
count++;
buffer2[count] = (byte)(((num8 - 0x111) << 4) & 240);
}
else
{
buffer2[count] = (byte)(((num8 - 1) << 4) & 240);
}
buffer2[count] = (byte)(buffer2[count] | ((byte)(((num6 - 1) >> 8) & 15)));
count++;
buffer2[count] = (byte)((num6 - 1) & 0xff);
count++;
}
num4++;
}
if (num4 > 0)
{
outstream.Write(buffer2, 0, count);
num2 += count;
}
}
return(num2);
}
public unsafe override int Compress(Stream instream, long inLength, Stream outstream)
{
// make sure the decompressed size fits in 3 bytes.
// There should be room for four bytes, however I'm not 100% sure if that can be used
// in every game, as it may not be a built-in function.
if (inLength > 0xFFFFFF)
{
throw new InputTooLargeException();
}
// use the other method if lookahead is enabled
if (lookAhead)
{
return(CompressWithLA(instream, inLength, outstream));
}
// save the input data in an array to prevent having to go back and forth in a file
byte[] indata = new byte[inLength];
int numReadBytes = instream.Read(indata, 0, (int)inLength);
if (numReadBytes != inLength)
{
throw new StreamTooShortException();
}
// write the compression header first
outstream.WriteByte(this.magicByte);
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
int compressedLength = 4;
fixed(byte *instart = &indata[0])
{
// we do need to buffer the output, as the first byte indicates which blocks are compressed.
// this version does not use a look-ahead, so we do not need to buffer more than 8 blocks at a time.
// (a block is at most 4 bytes long)
byte[] outbuffer = new byte[8 * 4 + 1];
outbuffer[0] = 0;
int bufferlength = 1, bufferedBlocks = 0;
int readBytes = 0;
while (readBytes < inLength)
{
#region If 8 blocks are bufferd, write them and reset the buffer
// we can only buffer 8 blocks at a time.
if (bufferedBlocks == 8)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
// reset the buffer
outbuffer[0] = 0;
bufferlength = 1;
bufferedBlocks = 0;
}
#endregion
// determine if we're dealing with a compressed or raw block.
// it is a compressed block when the next 3 or more bytes can be copied from
// somewhere in the set of already compressed bytes.
int disp;
int oldLength = Math.Min(readBytes, 0x1000);
int length = LZUtil.GetOccurrenceLength(instart + readBytes, (int)Math.Min(inLength - readBytes, 0x10110),
instart + readBytes - oldLength, oldLength, out disp);
// length not 3 or more? next byte is raw data
if (length < 3)
{
outbuffer[bufferlength++] = *(instart + (readBytes++));
}
else
{
// 3 or more bytes can be copied? next (length) bytes will be compressed into 2 bytes
readBytes += length;
// mark the next block as compressed
outbuffer[0] |= (byte)(1 << (7 - bufferedBlocks));
if (length > 0x110)
{
// case 1: 1(B CD E)(F GH) + (0x111)(0x1) = (LEN)(DISP)
outbuffer[bufferlength] = 0x10;
outbuffer[bufferlength] |= (byte)(((length - 0x111) >> 12) & 0x0F);
bufferlength++;
outbuffer[bufferlength] = (byte)(((length - 0x111) >> 4) & 0xFF);
bufferlength++;
outbuffer[bufferlength] = (byte)(((length - 0x111) << 4) & 0xF0);
}
else if (length > 0x10)
{
// case 0; 0(B C)(D EF) + (0x11)(0x1) = (LEN)(DISP)
outbuffer[bufferlength] = 0x00;
outbuffer[bufferlength] |= (byte)(((length - 0x111) >> 4) & 0x0F);
bufferlength++;
outbuffer[bufferlength] = (byte)(((length - 0x111) << 4) & 0xF0);
}
else
{
// case > 1: (A)(B CD) + (0x1)(0x1) = (LEN)(DISP)
outbuffer[bufferlength] = (byte)(((length - 1) << 4) & 0xF0);
}
// the last 1.5 bytes are always the disp
outbuffer[bufferlength] |= (byte)(((disp - 1) >> 8) & 0x0F);
bufferlength++;
outbuffer[bufferlength] = (byte)((disp - 1) & 0xFF);
bufferlength++;
}
bufferedBlocks++;
}
// copy the remaining blocks to the output
if (bufferedBlocks > 0)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
/*/ make the compressed file 4-byte aligned.
* while ((compressedLength % 4) != 0)
* {
* outstream.WriteByte(0);
* compressedLength++;
* }/**/
}
}
return(compressedLength);
}
/// <summary>
/// Compresses the given input stream with the LZ-Ovl compression, but compresses _forward_
/// instad of backwards.
/// </summary>
/// <param name="instream">The input stream to compress.</param>
/// <param name="inLength">The length of the input stream.</param>
/// <param name="outstream">The stream to write to.</param>
private unsafe int CompressNormal(Stream instream, long inLength, Stream outstream)
{
// make sure the decompressed size fits in 3 bytes.
// There should be room for four bytes, however I'm not 100% sure if that can be used
// in every game, as it may not be a built-in function.
if (inLength > 0xFFFFFF)
{
throw new InputTooLargeException();
}
// use the other method if lookahead is enabled
if (lookAhead)
{
return(CompressWithLA(instream, inLength, outstream));
}
// save the input data in an array to prevent having to go back and forth in a file
byte[] indata = new byte[inLength];
int numReadBytes = instream.Read(indata, 0, (int)inLength);
if (numReadBytes != inLength)
{
throw new StreamTooShortException();
}
int compressedLength = 0;
fixed(byte *instart = &indata[0])
{
// we do need to buffer the output, as the first byte indicates which blocks are compressed.
// this version does not use a look-ahead, so we do not need to buffer more than 8 blocks at a time.
byte[] outbuffer = new byte[8 * 2 + 1];
outbuffer[0] = 0;
int bufferlength = 1, bufferedBlocks = 0;
int readBytes = 0;
while (readBytes < inLength)
{
#region If 8 blocks are bufferd, write them and reset the buffer
// we can only buffer 8 blocks at a time.
if (bufferedBlocks == 8)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
// reset the buffer
outbuffer[0] = 0;
bufferlength = 1;
bufferedBlocks = 0;
}
#endregion
// determine if we're dealing with a compressed or raw block.
// it is a compressed block when the next 3 or more bytes can be copied from
// somewhere in the set of already compressed bytes.
int disp;
int oldLength = Math.Min(readBytes, 0x1001);
int length = LZUtil.GetOccurrenceLength(instart + readBytes, (int)Math.Min(inLength - readBytes, 0x12),
instart + readBytes - oldLength, oldLength, out disp);
// disp = 1 cannot be stored.
if (disp == 1)
{
length = 1;
}
// disp = 2 cannot be saved properly. use a too large disp instead.
// however since I'm not sure if that's actually how that's handled, don't compress instead.
else if (disp == 2)
{
length = 1;
/*if (readBytes < 0x1001)
* disp = readBytes + 1;
* else
* length = 1;/**/
}
// length not 3 or more? next byte is raw data
if (length < 3)
{
outbuffer[bufferlength++] = *(instart + (readBytes++));
}
else
{
// 3 or more bytes can be copied? next (length) bytes will be compressed into 2 bytes
readBytes += length;
// mark the next block as compressed
outbuffer[0] |= (byte)(1 << (7 - bufferedBlocks));
outbuffer[bufferlength] = (byte)(((length - 3) << 4) & 0xF0);
outbuffer[bufferlength] |= (byte)(((disp - 3) >> 8) & 0x0F);
bufferlength++;
outbuffer[bufferlength] = (byte)((disp - 3) & 0xFF);
bufferlength++;
}
bufferedBlocks++;
}
// copy the remaining blocks to the output
if (bufferedBlocks > 0)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
/*/ make the compressed file 4-byte aligned.
* while ((compressedLength % 4) != 0)
* {
* outstream.WriteByte(0);
* compressedLength++;
* }/**/
}
}
return(compressedLength);
}
public unsafe override int Compress(System.IO.Stream instream, long inLength, System.IO.Stream outstream)
{
// block type 0: stores at most 3+0xF = 0x12 = 18 bytes (in 2 bytes)
// block type 1: stores at most 2+0x3F = 0x41 = 65 bytes (in 1 byte)
// block type 2: 1 raw byte
// block type 3: 3 raw bytes
if (LookAhead)
{
return(CompressWithLA(instream, inLength, outstream));
}
// save the input data in an array to prevent having to go back and forth in a file
byte[] indata = new byte[inLength];
int numReadBytes = instream.Read(indata, 0, (int)inLength);
if (numReadBytes != inLength)
{
throw new StreamTooShortException();
}
// write the compression head first
outstream.WriteByte((byte)'L');
outstream.WriteByte((byte)'e');
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
outstream.WriteByte((byte)((inLength >> 24) & 0xFF));
int compressedLength = 6;
fixed(byte *instart = &indata[0])
{
// we do need to buffer the output, as the first byte indicates which blocks are compressed.
// this version does not use a look-ahead, so we do not need to buffer more than 4 blocks at a time.
// (a block is at most 3 bytes long)
byte[] outbuffer = new byte[4 * 3 + 1];
outbuffer[0] = 0;
int bufferlength = 1, bufferedBlocks = 0;
int readBytes = 0;
int cacheByte = -1;
while (readBytes < inLength)
{
#region If 4 blocks are bufferd, write them and reset the buffer
// we can only buffer 4 blocks at a time.
if (bufferedBlocks == 4)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
// reset the buffer
outbuffer[0] = 0;
bufferlength = 1;
bufferedBlocks = 0;
}
#endregion
// type 0: 3 <= len <= 18; 5 <= disp <= 0x1004
// type 1: 2 <= len <= 65; 1 <= disp <= 4
// type 2: 1 raw byte
// type 3: 3 raw bytes
// check if we can compress it using type 1 first (only 1 byte-long block)
int disp;
int oldLength = Math.Min(readBytes, 0x1004);
int length = LZUtil.GetOccurrenceLength(instart + readBytes, (int)Math.Min(inLength - readBytes, 65),
instart + readBytes - oldLength, oldLength, out disp, 1);
if (disp >= 1 && ((disp <= 4 && length >= 2) || (disp >= 5 && length >= 3)))
{
if (cacheByte >= 0)
{
// write a single raw byte block (the previous byte could not be the start of any compressed block)
outbuffer[bufferlength++] = (byte)(cacheByte & 0xFF);
outbuffer[0] |= (byte)(2 << (bufferedBlocks * 2));
cacheByte = -1;
bufferedBlocks++;
// the block set may be full; just retry this iteration.
continue;
}
if (disp >= 5)
{
#region compress using type 0
// type 0: store len/disp in 2 bytes:
// AB CD, with len = C + 3, disp = DAB + 5
// make sure we do not try to compress more than fits into the block
length = Math.Min(length, 0xF + 3);
readBytes += length;
outbuffer[bufferlength++] = (byte)((disp - 5) & 0xFF);
outbuffer[bufferlength] = (byte)(((disp - 5) >> 8) & 0xF);
outbuffer[bufferlength++] |= (byte)(((length - 3) & 0xF) << 4);
#endregion
}
else // 1 <= disp <= 4
{
#region compress using type 1
// type 1: store len/disp in 1 byte:
// ABCDEFGH, wih len = ABCDEF + 2, disp = GH + 1
readBytes += length;
outbuffer[bufferlength] = (byte)(((length - 2) << 2) & 0xFC);
outbuffer[bufferlength] |= (byte)((disp - 1) & 0x3);
bufferlength++;
outbuffer[0] |= (byte)(1 << (bufferedBlocks * 2));
#endregion
}
}
else
{
if (cacheByte < 0)
{
// first fail? remember byte, try to compress starting at next byte
cacheByte = *(instart + (readBytes++));
continue;
}
else
{
// 2 consecutive fails -> store 3 raw bytes (type 3) if possible.
if (inLength - readBytes >= 2)
{
outbuffer[bufferlength++] = (byte)(cacheByte & 0xFF);
outbuffer[bufferlength++] = *(instart + (readBytes++));
outbuffer[bufferlength++] = *(instart + (readBytes++));
outbuffer[0] |= (byte)(3 << (bufferedBlocks * 2));
cacheByte = -1;
}
else
{
// there are only two bytes remaining (incl. the cched byte)
// so write the cached byte first as single raw byte.
// keep the next/last byte as new cache, since the block buffer may be full.
outbuffer[bufferlength++] = (byte)(cacheByte & 0xFF);
outbuffer[0] |= (byte)(2 << (bufferedBlocks * 2));
cacheByte = *(instart + (readBytes++));
}
}
}
bufferedBlocks++;
}
// there may be one cache-byte left.
if (cacheByte >= 0)
{
// if the current set of blocks is full, empty it first
if (bufferedBlocks == 4)
{
#region empty block buffer
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
// reset the buffer
outbuffer[0] = 0;
bufferlength = 1;
bufferedBlocks = 0;
#endregion
}
outbuffer[bufferlength++] = (byte)(cacheByte & 0xFF);
cacheByte = -1;
outbuffer[0] |= (byte)(2 << (bufferedBlocks * 2));
bufferedBlocks++;
}
// copy any remaining blocks to the output
if (bufferedBlocks > 0)
{
outstream.Write(outbuffer, 0, bufferlength);
compressedLength += bufferlength;
}
}
return(compressedLength);
}
public unsafe int CompressNormal(Stream instream, long inLength, Stream outstream)
{
if (lookAhead)
{
return(CompressWithLA(instream, inLength, outstream));
}
byte[] buffer = new byte[inLength];
int num = instream.Read(buffer, 0, (int)inLength);
int num2 = 0;
fixed(byte *numRef = buffer)
{
byte[] buffer2 = new byte[0x11];
buffer2[0] = 0;
int count = 1;
int num4 = 0;
int num5 = 0;
while (num5 < inLength)
{
int num6;
if (num4 == 8)
{
outstream.Write(buffer2, 0, count);
num2 += count;
buffer2[0] = 0;
count = 1;
num4 = 0;
}
int oldLength = Math.Min(num5, 0x1001);
int num8 = LZUtil.GetOccurrenceLength(numRef + num5, (int)Math.Min((long)(inLength - num5), (long)0x12L), (numRef + num5) - oldLength, oldLength, out num6);
if (num6 == 1)
{
num8 = 1;
}
else if (num6 == 2)
{
num8 = 1;
}
if (num8 < 3)
{
buffer2[count++] = numRef[num5++];
}
else
{
num5 += num8;
buffer2[0] = (byte)(buffer2[0] | ((byte)(((int)1) << (7 - num4))));
buffer2[count] = (byte)(((num8 - 3) << 4) & 240);
buffer2[count] = (byte)(buffer2[count] | ((byte)(((num6 - 3) >> 8) & 15)));
count++;
buffer2[count] = (byte)((num6 - 3) & 0xff);
count++;
}
num4++;
}
if (num4 > 0)
{
outstream.Write(buffer2, 0, count);
num2 += count;
}
}
return(num2);
}