private static void Push(UInt16LE_E_L_OutputBitStream bitStream, bool bit, Stream destination, MemoryStream data)
{
if (bitStream.Push(bit))
{
byte[] bytes = data.ToArray();
destination.Write(bytes, 0, bytes.Length);
data.SetLength(0);
}
}
private static void Encode(Stream input, Stream output, Endianness endianness)
{
Action <Stream, ushort> write2;
OutputBitStream <ushort> bitStream;
if (endianness == Endianness.BigEndian)
{
write2 = Write2BE;
bitStream = new UInt16BE_E_L_OutputBitStream(output);
}
else
{
write2 = Write2LE;
bitStream = new UInt16LE_E_L_OutputBitStream(output);
}
// To unpack source into 2-byte words.
ushort[] words = new ushort[(input.Length - input.Position) / 2];
if (words.Length == 0)
{
throw new CompressionException(Properties.Resources.EmptySource);
}
// Frequency map.
SortedList <ushort, long> counts = new SortedList <ushort, long>();
// Presence map.
HashSet <ushort> elements = new HashSet <ushort>();
// Unpack source into array. Along the way, build frequency and presence maps.
ushort maskValue = 0;
{
byte[] buffer = new byte[2];
int i = 0, bytesRead;
while ((bytesRead = input.Read(buffer, 0, 2)) == 2)
{
ushort v = (ushort)(buffer[0] << 8 | buffer[1]);
maskValue |= v;
long count;
counts.TryGetValue(v, out count);
counts[v] = count + 1;
elements.Add(v);
words[i++] = v;
}
}
var writeBitfield = GetBitfieldWriter((byte)(maskValue >> 11));
byte packetLength = (byte)(Log2((ushort)(maskValue & 0x7ff)) + 1);
// Find the most common 2-byte value.
ushort commonValue = FindMostFrequentWord(counts);
// Find incrementing (not necessarily contiguous) runs.
// The original algorithm does this for all 65536 2-byte words, while
// this version only checks the 2-byte words actually in the file.
SortedList <ushort, long> runs = new SortedList <ushort, long>();
foreach (ushort element in elements)
{
ushort next = element;
long runLength = 0;
foreach (ushort word in words)
{
if (word == next)
{
++next;
++runLength;
}
}
runs[element] = runLength;
}
// Find the starting 2-byte value with the longest incrementing run.
ushort incrementingValue = FindMostFrequentWord(runs);
// Output header.
NeutralEndian.Write1(output, packetLength);
NeutralEndian.Write1(output, (byte)(maskValue >> 11));
write2(output, incrementingValue);
write2(output, commonValue);
// Output compressed data.
List <ushort> buf = new List <ushort>();
int pos = 0;
while (pos < words.Length)
{
ushort v = words[pos];
if (v == incrementingValue)
{
FlushBuffer(buf, bitStream, writeBitfield, packetLength);
ushort next = (ushort)(v + 1);
ushort count = 0;
for (int i = pos + 1; i < words.Length && count < 0xf; i++)
{
if (next != words[i])
{
break;
}
++next;
++count;
}
bitStream.Write((ushort)(0x00 | count), 6);
incrementingValue = next;
pos += count;
}
else if (v == commonValue)
{
FlushBuffer(buf, bitStream, writeBitfield, packetLength);
ushort count = 0;
for (int i = pos + 1; i < words.Length && count < 0xf; i++)
{
if (v != words[i])
{
break;
}
++count;
}
bitStream.Write((ushort)(0x10 | count), 6);
pos += count;
}
else
{
ushort next;
int delta;
if (pos + 1 < words.Length &&
(next = words[pos + 1]) != incrementingValue &&
((delta = (int)next - (int)v) == -1 || delta == 0 || delta == 1))
{
FlushBuffer(buf, bitStream, writeBitfield, packetLength);
ushort count = 1;
next = (ushort)(next + delta);
for (int i = pos + 2; i < words.Length && count < 0xf; i++)
{
if (next != words[i])
{
break;
}
// If the word is equal to the incrementing word value, stop this run early so we can use the
// incrementing value in the next iteration of the main loop.
if (words[i] == incrementingValue)
{
break;
}
next = (ushort)(next + delta);
++count;
}
if (delta == -1)
{
delta = 2;
}
delta |= 4;
delta <<= 4;
bitStream.Write((ushort)(delta | count), 7);
writeBitfield(bitStream, v);
bitStream.Write((ushort)(v & 0x7ff), packetLength);
pos += count;
}
else
{
if (buf.Count >= 0xf)
{
FlushBuffer(buf, bitStream, writeBitfield, packetLength);
}
buf.Add(v);
}
}
++pos;
}
FlushBuffer(buf, bitStream, writeBitfield, packetLength);
// Terminator
bitStream.Write(0x7f, 7);
bitStream.Flush(false);
}
private static void EncodeInternal(Stream destination, byte[] buffer, int pos, int size)
{
/*
* Here we create and populate the "LZSS graph":
*
* Each value in the uncompressed file forms a node in this graph.
* The various edges between these nodes represent LZSS matches.
*
* Using a shortest-path algorithm, these edges can be used to
* find the optimal combination of matches needed to produce the
* smallest possible file.
*
* The outputted array only contains one edge per node: the optimal
* one. This means, in order to produce the smallest file, you just
* have to traverse the graph from one edge to the next, encoding
* each match as you go along.
*/
LZSSGraphEdge[] node_meta_array = new LZSSGraphEdge[size + 1];
// Initialise the array
node_meta_array[0].cost = 0;
for (int i = 1; i < size + 1; ++i)
{
node_meta_array[i].cost = int.MaxValue;
}
// Find matches
for (int i = 0; i < size; ++i)
{
int max_read_ahead = Math.Min(0x100, size - i);
int max_read_behind = Math.Max(0, i - 0x2000);
// Search for dictionary matches
for (int j = i; j-- > max_read_behind;)
{
for (int k = 0; k < max_read_ahead; ++k)
{
if (buffer[pos + i + k] == buffer[pos + j + k])
{
int distance = i - j;
int length = k + 1;
// Get the cost of the match (or bail if it can't be compressed)
int cost;
if (length >= 2 && length <= 5 && distance <= 256)
{
cost = 2 + 2 + 8; // Descriptor bits, length bits, offset byte
}
else if (length >= 3 && length <= 9)
{
cost = 2 + 16; // Descriptor bits, offset/length bytes
}
else if (length >= 3)
{
cost = 2 + 16 + 8; // Descriptor bits, offset bytes, length byte
}
else
{
continue; // In the event a match cannot be compressed
}
// Update this node's optimal edge if this one is better
if (node_meta_array[i + k + 1].cost > node_meta_array[i].cost + cost)
{
node_meta_array[i + k + 1].cost = node_meta_array[i].cost + cost;
node_meta_array[i + k + 1].previous_node_index = i;
node_meta_array[i + k + 1].match_length = k + 1;
node_meta_array[i + k + 1].match_offset = j;
}
}
else
{
break;
}
}
}
// Do literal match
// Update this node's optimal edge if this one is better (or the same, since literal matches usually decode faster)
if (node_meta_array[i + 1].cost >= node_meta_array[i].cost + 1 + 8)
{
node_meta_array[i + 1].cost = node_meta_array[i].cost + 1 + 8;
node_meta_array[i + 1].previous_node_index = i;
node_meta_array[i + 1].match_length = 0;
}
}
// Reverse the edge link order, so the array can be traversed from start to end, rather than vice versa
node_meta_array[0].previous_node_index = int.MaxValue;
node_meta_array[size].next_node_index = int.MaxValue;
for (int node_index = size; node_meta_array[node_index].previous_node_index != int.MaxValue; node_index = node_meta_array[node_index].previous_node_index)
{
node_meta_array[node_meta_array[node_index].previous_node_index].next_node_index = node_index;
}
/*
* LZSS graph complete
*/
UInt16LE_E_L_OutputBitStream bitStream = new UInt16LE_E_L_OutputBitStream(destination);
MemoryStream data = new MemoryStream();
for (int node_index = 0; node_meta_array[node_index].next_node_index != int.MaxValue; node_index = node_meta_array[node_index].next_node_index)
{
int next_index = node_meta_array[node_index].next_node_index;
int length = node_meta_array[next_index].match_length;
int distance = next_index - node_meta_array[next_index].match_length - node_meta_array[next_index].match_offset;
if (length != 0)
{
if (length >= 2 && length <= 5 && distance <= 256)
{
Push(bitStream, false, destination, data);
Push(bitStream, false, destination, data);
Push(bitStream, ((length - 2) & 2) != 0, destination, data);
Push(bitStream, ((length - 2) & 1) != 0, destination, data);
NeutralEndian.Write1(data, (byte)-distance);
}
else if (length >= 3 && length <= 9)
{
Push(bitStream, false, destination, data);
Push(bitStream, true, destination, data);
NeutralEndian.Write1(data, (byte)(-distance & 0xFF));
NeutralEndian.Write1(data, (byte)(((-distance >> (8 - 3)) & 0xF8) | ((length - 2) & 7)));
}
else //if (length >= 3)
{
Push(bitStream, false, destination, data);
Push(bitStream, true, destination, data);
NeutralEndian.Write1(data, (byte)(-distance & 0xFF));
NeutralEndian.Write1(data, (byte)((-distance >> (8 - 3)) & 0xF8));
NeutralEndian.Write1(data, (byte)(length - 1));
}
}
else
{
Push(bitStream, true, destination, data);
NeutralEndian.Write1(data, buffer[pos + node_index]);
}
}
Push(bitStream, false, destination, data);
Push(bitStream, true, destination, data);
// If the bit stream was just flushed, write an empty bit stream that will be read just before the end-of-data
// sequence below.
if (!bitStream.HasWaitingBits)
{
NeutralEndian.Write1(data, 0);
NeutralEndian.Write1(data, 0);
}
NeutralEndian.Write1(data, 0);
NeutralEndian.Write1(data, 0xF0);
NeutralEndian.Write1(data, 0);
bitStream.Flush(true);
byte[] bytes = data.ToArray();
destination.Write(bytes, 0, bytes.Length);
}
private static void EncodeInternal(Stream destination, byte[] buffer, long pos, long slidingWindow, long recLength, long size)
{
UInt16LE_E_L_OutputBitStream bitStream = new UInt16LE_E_L_OutputBitStream(destination);
MemoryStream data = new MemoryStream();
if (size > 0)
{
long bPointer = 1, iOffset = 0;
bitStream.Push(true);
NeutralEndian.Write1(data, buffer[pos]);
while (bPointer < size)
{
long iCount = Math.Min(recLength, size - bPointer);
long iMax = Math.Max(bPointer - slidingWindow, 0);
long k = 1;
long i = bPointer - 1;
do
{
long j = 0;
while (buffer[pos + i + j] == buffer[pos + bPointer + j])
{
if (++j >= iCount)
{
break;
}
}
if (j > k)
{
k = j;
iOffset = i;
}
} while (i-- > iMax);
iCount = k;
if (iCount == 1)
{
Push(bitStream, true, destination, data);
NeutralEndian.Write1(data, buffer[pos + bPointer]);
}
else if (iCount == 2 && bPointer - iOffset > 256)
{
Push(bitStream, true, destination, data);
NeutralEndian.Write1(data, buffer[pos + bPointer]);
--iCount;
}
else if (iCount < 6 && bPointer - iOffset <= 256)
{
Push(bitStream, false, destination, data);
Push(bitStream, false, destination, data);
Push(bitStream, (((iCount - 2) >> 1) & 1) != 0, destination, data);
Push(bitStream, ((iCount - 2) & 1) != 0, destination, data);
NeutralEndian.Write1(data, (byte)(~(bPointer - iOffset - 1)));
}
else
{
Push(bitStream, false, destination, data);
Push(bitStream, true, destination, data);
long off = bPointer - iOffset - 1;
ushort info = (ushort)(~((off << 8) | (off >> 5)) & 0xFFF8);
if (iCount < 10) // iCount - 2 < 8
{
info |= (ushort)(iCount - 2);
BigEndian.Write2(data, info);
}
else
{
BigEndian.Write2(data, info);
NeutralEndian.Write1(data, (byte)(iCount - 1));
}
}
bPointer += iCount;
}
}
Push(bitStream, false, destination, data);
Push(bitStream, true, destination, data);
// If the bit stream was just flushed, write an empty bit stream that will be read just before the end-of-data
// sequence below.
if (!bitStream.HasWaitingBits)
{
NeutralEndian.Write1(data, 0);
NeutralEndian.Write1(data, 0);
}
NeutralEndian.Write1(data, 0);
NeutralEndian.Write1(data, 0xF0);
NeutralEndian.Write1(data, 0);
bitStream.Flush(true);
byte[] bytes = data.ToArray();
destination.Write(bytes, 0, bytes.Length);
}