/// <summary>
/// Applies Huffman compression with a datablock size of 8 bits.
/// </summary>
/// <param name="instream">The stream to compress.</param>
/// <param name="inLength">The length of the input stream.</param>
/// <param name="outstream">The stream to write the decompressed data to.</param>
/// <returns>The size of the decompressed data.</returns>
private int Compress8(Stream instream, long inLength, Stream outstream)
{
if (inLength > 0xFFFFFF)
{
throw new InputTooLargeException();
}
// cache the input, as we need to build a frequency table
byte[] inputData = new byte[inLength];
instream.Read(inputData, 0, (int)inLength);
// build that frequency table.
int[] frequencies = new int[0x100];
for (int i = 0; i < inLength; i++)
{
frequencies[inputData[i]]++;
}
#region Build the Huffman tree
SimpleReversedPrioQueue <int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
SimpleReversedPrioQueue <int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
int nodeCount = 0;
// make all leaf nodes, and put them in the leaf queue. Also save them for later use.
HuffTreeNode[] leaves = new HuffTreeNode[0x100];
for (int i = 0; i < 0x100; i++)
{
// there is no need to store leaves that are not used
if (frequencies[i] == 0)
{
continue;
}
HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
leaves[i] = node;
leafQueue.Enqueue(frequencies[i], node);
nodeCount++;
}
while (leafQueue.Count + nodeQueue.Count > 1)
{
// get the two nodes with the lowest priority.
HuffTreeNode one = null, two = null;
int onePrio, twoPrio;
one = GetLowest(leafQueue, nodeQueue, out onePrio);
two = GetLowest(leafQueue, nodeQueue, out twoPrio);
// give those two a common parent, and put that node in the node queue
HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
nodeQueue.Enqueue(onePrio + twoPrio, newNode);
nodeCount++;
}
int rootPrio;
HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
// set the depth of all nodes in the tree, such that we know for each leaf how long
// its codeword is.
root.Depth = 0;
#endregion
// now that we have a tree, we can write that tree and follow with the data.
// write the compression header first
outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
int compressedLength = 4;
#region write the tree
outstream.WriteByte((byte)((nodeCount - 1) / 2));
compressedLength++;
// use a breadth-first traversal to store the tree, such that we do not need to store/calculate the side of each sub-tree.
LinkedList <HuffTreeNode> printQueue = new LinkedList <HuffTreeNode>();
printQueue.AddLast(root);
while (printQueue.Count > 0)
{
HuffTreeNode node = printQueue.First.Value;
printQueue.RemoveFirst();
if (node.IsData)
{
outstream.WriteByte(node.Data);
}
else
{
// bits 0-5: 'offset' = # nodes in queue left
// bit 6: node1 end flag
// bit 7: node0 end flag
byte data = (byte)(printQueue.Count / 2);
data = (byte)(data & 0x3F);
if (node.Child0.IsData)
{
data |= 0x80;
}
if (node.Child1.IsData)
{
data |= 0x40;
}
outstream.WriteByte(data);
printQueue.AddLast(node.Child0);
printQueue.AddLast(node.Child1);
}
compressedLength++;
}
#endregion
#region write the data
// the codewords are stored in blocks of 32 bits
uint datablock = 0;
byte bitsLeftToWrite = 32;
for (int i = 0; i < inLength; i++)
{
byte data = inputData[i];
HuffTreeNode node = leaves[data];
// the depth of the node is the length of the codeword required to encode the byte
int depth = node.Depth;
bool[] path = new bool[depth];
for (int d = 0; d < depth; d++)
{
path[depth - d - 1] = node.IsChild1;
node = node.Parent;
}
for (int d = 0; d < depth; d++)
{
if (bitsLeftToWrite == 0)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
datablock = 0;
bitsLeftToWrite = 32;
}
bitsLeftToWrite--;
if (path[d])
{
datablock |= (uint)(1 << bitsLeftToWrite);
}
// no need to OR the buffer with 0 if it is child0
}
}
// write the partly filled data block as well
if (bitsLeftToWrite != 32)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
}
#endregion
return(compressedLength);
}
/// <summary>
/// Applies Huffman compression with a datablock size of 8 bits.
/// </summary>
/// <param name="instream">The stream to compress.</param>
/// <param name="inLength">The length of the input stream.</param>
/// <param name="outstream">The stream to write the decompressed data to.</param>
/// <returns>The size of the decompressed data.</returns>
private int Compress8(Stream instream, long inLength, Stream outstream)
{
if (inLength > 0xFFFFFF)
throw new InputTooLargeException();
// cache the input, as we need to build a frequency table
byte[] inputData = new byte[inLength];
instream.Read(inputData, 0, (int)inLength);
// build that frequency table.
int[] frequencies = new int[0x100];
for (int i = 0; i < inLength; i++)
frequencies[inputData[i]]++;
#region Build the Huffman tree
SimpleReversedPrioQueue<int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
SimpleReversedPrioQueue<int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
int nodeCount = 0;
// make all leaf nodes, and put them in the leaf queue. Also save them for later use.
HuffTreeNode[] leaves = new HuffTreeNode[0x100];
for (int i = 0; i < 0x100; i++)
{
// there is no need to store leaves that are not used
if (frequencies[i] == 0)
continue;
HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
leaves[i] = node;
leafQueue.Enqueue(frequencies[i], node);
nodeCount++;
}
while (leafQueue.Count + nodeQueue.Count > 1)
{
// get the two nodes with the lowest priority.
HuffTreeNode one = null, two = null;
int onePrio, twoPrio;
one = GetLowest(leafQueue, nodeQueue, out onePrio);
two = GetLowest(leafQueue, nodeQueue, out twoPrio);
// give those two a common parent, and put that node in the node queue
HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
nodeQueue.Enqueue(onePrio + twoPrio, newNode);
nodeCount++;
}
int rootPrio;
HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
// set the depth of all nodes in the tree, such that we know for each leaf how long
// its codeword is.
root.Depth = 0;
#endregion
// now that we have a tree, we can write that tree and follow with the data.
// write the compression header first
outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
int compressedLength = 4;
#region write the tree
outstream.WriteByte((byte)((nodeCount - 1) / 2));
compressedLength++;
// use a breadth-first traversal to store the tree, such that we do not need to store/calculate the side of each sub-tree.
LinkedList<HuffTreeNode> printQueue = new LinkedList<HuffTreeNode>();
printQueue.AddLast(root);
while (printQueue.Count > 0)
{
HuffTreeNode node = printQueue.First.Value;
printQueue.RemoveFirst();
if (node.IsData)
{
outstream.WriteByte(node.Data);
}
else
{
// bits 0-5: 'offset' = # nodes in queue left
// bit 6: node1 end flag
// bit 7: node0 end flag
byte data = (byte)(printQueue.Count / 2);
data = (byte)(data & 0x3F);
if (node.Child0.IsData)
data |= 0x80;
if (node.Child1.IsData)
data |= 0x40;
outstream.WriteByte(data);
printQueue.AddLast(node.Child0);
printQueue.AddLast(node.Child1);
}
compressedLength++;
}
#endregion
#region write the data
// the codewords are stored in blocks of 32 bits
uint datablock = 0;
byte bitsLeftToWrite = 32;
for (int i = 0; i < inLength; i++)
{
byte data = inputData[i];
HuffTreeNode node = leaves[data];
// the depth of the node is the length of the codeword required to encode the byte
int depth = node.Depth;
bool[] path = new bool[depth];
for (int d = 0; d < depth; d++)
{
path[depth - d - 1] = node.IsChild1;
node = node.Parent;
}
for (int d = 0; d < depth; d++)
{
if (bitsLeftToWrite == 0)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
datablock = 0;
bitsLeftToWrite = 32;
}
bitsLeftToWrite--;
if (path[d])
datablock |= (uint)(1 << bitsLeftToWrite);
// no need to OR the buffer with 0 if it is child0
}
}
// write the partly filled data block as well
if (bitsLeftToWrite != 32)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
}
#endregion
return compressedLength;
}
/// <summary>
/// Applies Huffman compression with a datablock size of 8 bits.
/// </summary>
/// <param name="instream">The stream to compress.</param>
/// <param name="inLength">The length of the input stream.</param>
/// <param name="outstream">The stream to write the decompressed data to.</param>
/// <returns>The size of the decompressed data.</returns>
public override int Compress(Stream instream, long inLength, Stream outstream)
{
if (inLength > 0xFFFFFF)
{
throw new InputTooLargeException();
}
// cache the input, as we need to build a frequency table
byte[] inputData = new byte[inLength];
instream.Read(inputData, 0, (int)inLength);
// build that frequency table.
int[] frequencies = new int[0x100];
for (int i = 0; i < inLength; i++)
{
frequencies[inputData[i]]++;
}
#region Build the Huffman tree
SimpleReversedPrioQueue <int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
SimpleReversedPrioQueue <int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
int nodeCount = 0;
// make all leaf nodes, and put them in the leaf queue. Also save them for later use.
HuffTreeNode[] leaves = new HuffTreeNode[0x100];
for (int i = 0; i < 0x100; i++)
{
// there is no need to store leaves that are not used
if (frequencies[i] == 0)
{
continue;
}
HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
leaves[i] = node;
leafQueue.Enqueue(frequencies[i], node);
nodeCount++;
}
while (leafQueue.Count + nodeQueue.Count > 1)
{
// get the two nodes with the lowest priority.
HuffTreeNode one = null, two = null;
int onePrio, twoPrio;
one = GetLowest(leafQueue, nodeQueue, out onePrio);
two = GetLowest(leafQueue, nodeQueue, out twoPrio);
// give those two a common parent, and put that node in the node queue
HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
nodeQueue.Enqueue(onePrio + twoPrio, newNode);
nodeCount++;
}
int rootPrio;
HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
// set the depth of all nodes in the tree, such that we know for each leaf how long
// its codeword is.
root.Depth = 0;
#endregion
// now that we have a tree, we can write that tree and follow with the data.
// write the compression header first
outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
int compressedLength = 4;
#region write the tree
outstream.WriteByte((byte)((nodeCount - 1) / 2));
compressedLength++;
// use a breadth-first traversal to store the tree, such that we do not need to store/calculate the size of each sub-tree.
// NO! BF results in an ordering that may overflow the offset field.
// find the BF order of all nodes that have two leaves as children. We're going to insert them in an array in reverse BF order,
// inserting the parent whenever both children have been inserted.
LinkedList <HuffTreeNode> leafStemQueue = new LinkedList <HuffTreeNode>();
#region fill the leaf queue; first->last will be reverse BF
LinkedList <HuffTreeNode> nodeCodeStack = new LinkedList <HuffTreeNode>();
nodeCodeStack.AddLast(root);
while (nodeCodeStack.Count > 0)
{
HuffTreeNode node = nodeCodeStack.First.Value;
nodeCodeStack.RemoveFirst();
if (node.IsData)
{
continue;
}
if (node.Child0.IsData && node.Child1.IsData)
{
leafStemQueue.AddFirst(node);
}
else
{
nodeCodeStack.AddLast(node.Child0);
nodeCodeStack.AddLast(node.Child1);
}
}
#endregion
HuffTreeNode[] nodeArray = new HuffTreeNode[0x1FF]; // this array does not contain the leaves themselves!
while (leafStemQueue.Count > 0)
{
Insert(leafStemQueue.First.Value, nodeArray, 0x3F + 1);
leafStemQueue.RemoveFirst();
}
// update the indices to ignore all gaps
int nodeIndex = 0;
for (int i = 0; i < nodeArray.Length; i++)
{
if (nodeArray[i] != null)
{
nodeArray[i].index = nodeIndex++;
}
}
// write the nodes in their given order. However when 'writing' a node, write the data of its children instead.
// the root node is always the first node.
byte rootData = 0;
if (root.Child0.IsData)
{
rootData |= 0x80;
}
if (root.Child1.IsData)
{
rootData |= 0x40;
}
outstream.WriteByte(rootData); compressedLength++;
for (int i = 0; i < nodeArray.Length; i++)
{
if (nodeArray[i] != null)
{
// nodes in this array are never data!
HuffTreeNode node0 = nodeArray[i].Child0;
if (node0.IsData)
{
outstream.WriteByte(node0.Data);
}
else
{
int offset = node0.index - nodeArray[i].index - 1;
if (offset > 0x3F)
{
throw new Exception("Offset overflow!");
}
byte data = (byte)offset;
if (node0.Child0.IsData)
{
data |= 0x80;
}
if (node0.Child1.IsData)
{
data |= 0x40;
}
outstream.WriteByte(data);
}
HuffTreeNode node1 = nodeArray[i].Child1;
if (node1.IsData)
{
outstream.WriteByte(node1.Data);
}
else
{
int offset = node1.index - nodeArray[i].index - 1;
if (offset > 0x3F)
{
throw new Exception("Offset overflow!");
}
byte data = (byte)offset;
if (node0.Child0.IsData)
{
data |= 0x80;
}
if (node0.Child1.IsData)
{
data |= 0x40;
}
outstream.WriteByte(data);
}
compressedLength += 2;
}
}
#endregion
#region write the data
// the codewords are stored in blocks of 32 bits
uint datablock = 0;
byte bitsLeftToWrite = 32;
for (int i = 0; i < inLength; i++)
{
byte data = inputData[i];
HuffTreeNode node = leaves[data];
// the depth of the node is the length of the codeword required to encode the byte
int depth = node.Depth;
bool[] path = new bool[depth];
for (int d = 0; d < depth; d++)
{
path[depth - d - 1] = node.IsChild1;
node = node.Parent;
}
for (int d = 0; d < depth; d++)
{
if (bitsLeftToWrite == 0)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
datablock = 0;
bitsLeftToWrite = 32;
}
bitsLeftToWrite--;
if (path[d])
{
datablock |= (uint)(1 << bitsLeftToWrite);
}
// no need to OR the buffer with 0 if it is child0
}
}
// write the partly filled data block as well
if (bitsLeftToWrite != 32)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
}
#endregion
return(compressedLength);
}
/// <summary>
/// Applies Huffman compression with a datablock size of 8 bits.
/// </summary>
/// <param name="instream">The stream to compress.</param>
/// <param name="inLength">The length of the input stream.</param>
/// <param name="outstream">The stream to write the decompressed data to.</param>
/// <returns>The size of the decompressed data.</returns>
public override int Compress(Stream instream, long inLength, Stream outstream)
{
if (inLength > 0xFFFFFF)
throw new InputTooLargeException();
// cache the input, as we need to build a frequency table
byte[] inputData = new byte[inLength];
instream.Read(inputData, 0, (int)inLength);
// build that frequency table.
int[] frequencies = new int[0x100];
for (int i = 0; i < inLength; i++)
frequencies[inputData[i]]++;
#region Build the Huffman tree
SimpleReversedPrioQueue<int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
SimpleReversedPrioQueue<int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
int nodeCount = 0;
// make all leaf nodes, and put them in the leaf queue. Also save them for later use.
HuffTreeNode[] leaves = new HuffTreeNode[0x100];
for (int i = 0; i < 0x100; i++)
{
// there is no need to store leaves that are not used
if (frequencies[i] == 0)
continue;
HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
leaves[i] = node;
leafQueue.Enqueue(frequencies[i], node);
nodeCount++;
}
while (leafQueue.Count + nodeQueue.Count > 1)
{
// get the two nodes with the lowest priority.
HuffTreeNode one = null, two = null;
int onePrio, twoPrio;
one = GetLowest(leafQueue, nodeQueue, out onePrio);
two = GetLowest(leafQueue, nodeQueue, out twoPrio);
// give those two a common parent, and put that node in the node queue
HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
nodeQueue.Enqueue(onePrio + twoPrio, newNode);
nodeCount++;
}
int rootPrio;
HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
// set the depth of all nodes in the tree, such that we know for each leaf how long
// its codeword is.
root.Depth = 0;
#endregion
// now that we have a tree, we can write that tree and follow with the data.
// write the compression header first
outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
outstream.WriteByte((byte)(inLength & 0xFF));
outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
outstream.WriteByte((byte)((inLength >> 16) & 0xFF));
int compressedLength = 4;
#region write the tree
outstream.WriteByte((byte)((nodeCount - 1) / 2));
compressedLength++;
// use a breadth-first traversal to store the tree, such that we do not need to store/calculate the size of each sub-tree.
// NO! BF results in an ordering that may overflow the offset field.
// find the BF order of all nodes that have two leaves as children. We're going to insert them in an array in reverse BF order,
// inserting the parent whenever both children have been inserted.
LinkedList<HuffTreeNode> leafStemQueue = new LinkedList<HuffTreeNode>();
#region fill the leaf queue; first->last will be reverse BF
LinkedList<HuffTreeNode> nodeCodeStack = new LinkedList<HuffTreeNode>();
nodeCodeStack.AddLast(root);
while (nodeCodeStack.Count > 0)
{
HuffTreeNode node = nodeCodeStack.First.Value;
nodeCodeStack.RemoveFirst();
if (node.IsData)
continue;
if (node.Child0.IsData && node.Child1.IsData)
{
leafStemQueue.AddFirst(node);
}
else
{
nodeCodeStack.AddLast(node.Child0);
nodeCodeStack.AddLast(node.Child1);
}
}
#endregion
HuffTreeNode[] nodeArray = new HuffTreeNode[0x1FF]; // this array does not contain the leaves themselves!
while (leafStemQueue.Count > 0)
{
Insert(leafStemQueue.First.Value, nodeArray, 0x3F + 1);
leafStemQueue.RemoveFirst();
}
// update the indices to ignore all gaps
int nodeIndex = 0;
for (int i = 0; i < nodeArray.Length; i++)
{
if (nodeArray[i] != null)
nodeArray[i].index = nodeIndex++;
}
// write the nodes in their given order. However when 'writing' a node, write the data of its children instead.
// the root node is always the first node.
byte rootData = 0;
if (root.Child0.IsData)
rootData |= 0x80;
if (root.Child1.IsData)
rootData |= 0x40;
outstream.WriteByte(rootData); compressedLength++;
for (int i = 0; i < nodeArray.Length; i++)
{
if (nodeArray[i] != null)
{
// nodes in this array are never data!
HuffTreeNode node0 = nodeArray[i].Child0;
if (node0.IsData)
outstream.WriteByte(node0.Data);
else
{
int offset = node0.index - nodeArray[i].index - 1;
if (offset > 0x3F)
throw new Exception("Offset overflow!");
byte data = (byte)offset;
if (node0.Child0.IsData)
data |= 0x80;
if (node0.Child1.IsData)
data |= 0x40;
outstream.WriteByte(data);
}
HuffTreeNode node1 = nodeArray[i].Child1;
if (node1.IsData)
outstream.WriteByte(node1.Data);
else
{
int offset = node1.index - nodeArray[i].index - 1;
if (offset > 0x3F)
throw new Exception("Offset overflow!");
byte data = (byte)offset;
if (node0.Child0.IsData)
data |= 0x80;
if (node0.Child1.IsData)
data |= 0x40;
outstream.WriteByte(data);
}
compressedLength += 2;
}
}
#endregion
#region write the data
// the codewords are stored in blocks of 32 bits
uint datablock = 0;
byte bitsLeftToWrite = 32;
for (int i = 0; i < inLength; i++)
{
byte data = inputData[i];
HuffTreeNode node = leaves[data];
// the depth of the node is the length of the codeword required to encode the byte
int depth = node.Depth;
bool[] path = new bool[depth];
for (int d = 0; d < depth; d++)
{
path[depth - d - 1] = node.IsChild1;
node = node.Parent;
}
for (int d = 0; d < depth; d++)
{
if (bitsLeftToWrite == 0)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
datablock = 0;
bitsLeftToWrite = 32;
}
bitsLeftToWrite--;
if (path[d])
datablock |= (uint)(1 << bitsLeftToWrite);
// no need to OR the buffer with 0 if it is child0
}
}
// write the partly filled data block as well
if (bitsLeftToWrite != 32)
{
outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
compressedLength += 4;
}
#endregion
return compressedLength;
}