Exemplo n.º 1
0
            public HuffTreeNode(HUFF_STREAM stream, bool isData, long relOffset, long maxStreamPos)
            {
                if (stream.p >= maxStreamPos)
                {
                    return;
                }
                int readData = stream.ReadByte();

                this.data = (byte)readData;

                this.isData = isData;

                if (!this.isData)
                {
                    int  offset     = this.data & 0x3F;
                    bool zeroIsData = (this.data & 0x80) > 0;
                    bool oneIsData  = (this.data & 0x40) > 0;

                    long zeroRelOffset = (relOffset ^ (relOffset & 1)) + (offset * 2) + 2;

                    int currStreamPos = stream.p;
                    stream.p   += (int)(zeroRelOffset - relOffset) - 1;
                    this.child0 = new HuffTreeNode(stream, zeroIsData, zeroRelOffset, maxStreamPos);
                    this.child1 = new HuffTreeNode(stream, oneIsData, zeroRelOffset + 1, maxStreamPos);

                    stream.p = currStreamPos;
                }
            }
Exemplo n.º 2
0
            /// <summary>
            /// Creates a new node in the Huffman tree.
            /// </summary>
            /// <param name="stream">The stream to read from. It is assumed that there is (at least)
            /// one more byte available to read.</param>
            /// <param name="isData">If this node is a data-node.</param>
            /// <param name="relOffset">The offset of this node in the source data, relative to the start
            /// of the compressed file.</param>
            /// <param name="maxStreamPos">The indicated end of the huffman tree. If the stream is past
            /// this position, the tree is invalid.</param>
            public HuffTreeNode(Stream stream, bool isData, long relOffset, long maxStreamPos)
            {
                /*
                 * Tree Table (list of 8bit nodes, starting with the root node)
                 *  Root Node and Non-Data-Child Nodes are:
                 *  Bit0-5   Offset to next child node,
                 *          Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2
                 *          Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1
                 *  Bit6     Node1 End Flag (1=Next child node is data)
                 *  Bit7     Node0 End Flag (1=Next child node is data)
                 *  Data nodes are (when End Flag was set in parent node):
                 *  Bit0-7   Data (upper bits should be zero if Data Size is less than 8)
                 */

                if (stream.Position >= maxStreamPos)
                {
                    // this happens when part of the tree is unused.
                    this.isFilled = false;
                    return;
                }
                this.isFilled = true;
                int readData = stream.ReadByte();

                if (readData < 0)
                {
                    throw new StreamTooShortException();
                }
                this.data = (byte)readData;

                this.isData = isData;

                if (!this.isData)
                {
                    int  offset     = this.data & 0x3F;
                    bool zeroIsData = (this.data & 0x80) > 0;
                    bool oneIsData  = (this.data & 0x40) > 0;

                    // off AND NOT 1 == off XOR (off AND 1)
                    long zeroRelOffset = (relOffset ^ (relOffset & 1)) + offset * 2 + 2;

                    long currStreamPos = stream.Position;
                    // position the stream right before the 0-node
                    stream.Position += (zeroRelOffset - relOffset) - 1;
                    // read the 0-node
                    this.child0        = new HuffTreeNode(stream, zeroIsData, zeroRelOffset, maxStreamPos);
                    this.child0.Parent = this;
                    // the 1-node is directly behind the 0-node
                    this.child1        = new HuffTreeNode(stream, oneIsData, zeroRelOffset + 1, maxStreamPos);
                    this.child1.Parent = this;

                    // reset the stream position to right behind this node's data
                    stream.Position = currStreamPos;
                }
            }
Exemplo n.º 3
0
 /// <summary>
 /// Manually creates a new node for a huffman tree.
 /// </summary>
 /// <param name="data">The data for this node.</param>
 /// <param name="isData">If this node represents data.</param>
 /// <param name="child0">The child of this node on the 0 side.</param>
 /// <param name="child1">The child of this node on the 1 side.</param>
 public HuffTreeNode(byte data, bool isData, HuffTreeNode child0, HuffTreeNode child1)
 {
     this.data     = data;
     this.isData   = isData;
     this.child0   = child0;
     this.child1   = child1;
     this.isFilled = true;
     if (!isData)
     {
         this.child0.Parent = this;
         this.child1.Parent = this;
     }
 }
Exemplo n.º 4
0
        /// <summary>
        /// Shifts the node at the given index one to the right.
        /// If the distance between parent and child becomes too large due to this shift, the parent is shifted as well.
        /// </summary>
        /// <param name="array">The array to shift the node in.</param>
        /// <param name="idx">The index of the node to shift.</param>
        /// <param name="maxOffset">The maximum distance between parent and children.</param>
        private void ShiftRight(HuffTreeNode[] array, int idx, int maxOffset)
        {
            HuffTreeNode node = array[idx];

            if (array[idx + 1] != null)
            {
                ShiftRight(array, idx + 1, maxOffset);
            }
            if (node.Parent.index > 0 && node.index - maxOffset + 1 > node.Parent.index)
            {
                ShiftRight(array, node.Parent.index, maxOffset);
            }
            if (node != array[idx])
            {
                return; // already done indirectly.
            }
            array[idx + 1] = array[idx];
            array[idx]     = null;
            node.index++;
        }
Exemplo n.º 5
0
            internal bool getValue(LinkedListNode <byte> code, out int value)
            {
                value = data;
                if (code == null)
                {
                    return(node0 == null && node1 == null && data >= 0);
                }

                if (code.Value > 1)
                {
                    throw new Exception(String.Format("the list should be a list of bytes < 2. got:{0:g}", code.Value));
                }

                byte         c = code.Value;
                bool         retVal;
                HuffTreeNode n = c == 0 ? node0 : node1;

                retVal = n != null && n.getValue(code.Previous, out value);
                return(retVal);
            }
Exemplo n.º 6
0
            internal void parseData(BinaryReader br)
            {
                node0 = new HuffTreeNode();
                node1 = new HuffTreeNode();
                long currPos = br.BaseStream.Position;
                byte b = br.ReadByte();
                long offset = b & 0x3F;
                bool end0 = (b & 0x80) > 0, end1 = (b & 0x40) > 0;

                br.BaseStream.Position = (currPos - (currPos & 1)) + offset * 2 + 2;
                if (br.BaseStream.Position < maxInpos)
                {
                    if (end0)
                    {
                        node0.data = br.ReadByte();
                    }
                    else
                    {
                        node0.parseData(br);
                    }
                }

                br.BaseStream.Position = (currPos - (currPos & 1)) + offset * 2 + 2 + 1;
                if (br.BaseStream.Position < maxInpos)
                {
                    if (end1)
                    {
                        node1.data = br.ReadByte();
                    }
                    else
                    {
                        node1.parseData(br);
                    }
                }

                br.BaseStream.Position = currPos;
            }
Exemplo n.º 7
0
        /// <summary>
        /// Applies Huffman compression with a datablock size of 8 bits.
        /// </summary>
        /// <param name="instream">The stream to compress.</param>
        /// <param name="inLength">The length of the input stream.</param>
        /// <param name="outstream">The stream to write the decompressed data to.</param>
        /// <returns>The size of the decompressed data.</returns>
        public override int Compress(Stream instream, long inLength, Stream outstream)
        {
            if (inLength > 0xFFFFFF)
            {
                throw new InputTooLargeException();
            }

            // cache the input, as we need to build a frequency table
            byte[] inputData = new byte[inLength];
            instream.Read(inputData, 0, (int)inLength);

            // build that frequency table.
            int[] frequencies = new int[0x100];
            for (int i = 0; i < inLength; i++)
            {
                frequencies[inputData[i]]++;
            }

            #region Build the Huffman tree

            SimpleReversedPrioQueue <int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
            SimpleReversedPrioQueue <int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
            int nodeCount = 0;
            // make all leaf nodes, and put them in the leaf queue. Also save them for later use.
            HuffTreeNode[] leaves = new HuffTreeNode[0x100];
            for (int i = 0; i < 0x100; i++)
            {
                // there is no need to store leaves that are not used
                if (frequencies[i] == 0)
                {
                    continue;
                }
                HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
                leaves[i] = node;
                leafQueue.Enqueue(frequencies[i], node);
                nodeCount++;
            }

            while (leafQueue.Count + nodeQueue.Count > 1)
            {
                // get the two nodes with the lowest priority.
                HuffTreeNode one = null, two = null;
                int          onePrio, twoPrio;
                one = GetLowest(leafQueue, nodeQueue, out onePrio);
                two = GetLowest(leafQueue, nodeQueue, out twoPrio);

                // give those two a common parent, and put that node in the node queue
                HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
                nodeQueue.Enqueue(onePrio + twoPrio, newNode);
                nodeCount++;
            }
            int          rootPrio;
            HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
            // set the depth of all nodes in the tree, such that we know for each leaf how long
            // its codeword is.
            root.Depth = 0;

            #endregion

            // now that we have a tree, we can write that tree and follow with the data.

            // write the compression header first
            outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
            outstream.WriteByte((byte)(inLength & 0xFF));
            outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
            outstream.WriteByte((byte)((inLength >> 16) & 0xFF));

            int compressedLength = 4;

            #region write the tree

            outstream.WriteByte((byte)((nodeCount - 1) / 2));
            compressedLength++;

            // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the size of each sub-tree.
            // NO! BF results in an ordering that may overflow the offset field.

            // find the BF order of all nodes that have two leaves as children. We're going to insert them in an array in reverse BF order,
            // inserting the parent whenever both children have been inserted.

            LinkedList <HuffTreeNode> leafStemQueue = new LinkedList <HuffTreeNode>();

            #region fill the leaf queue; first->last will be reverse BF
            LinkedList <HuffTreeNode> nodeCodeStack = new LinkedList <HuffTreeNode>();
            nodeCodeStack.AddLast(root);
            while (nodeCodeStack.Count > 0)
            {
                HuffTreeNode node = nodeCodeStack.First.Value;
                nodeCodeStack.RemoveFirst();
                if (node.IsData)
                {
                    continue;
                }
                if (node.Child0.IsData && node.Child1.IsData)
                {
                    leafStemQueue.AddFirst(node);
                }
                else
                {
                    nodeCodeStack.AddLast(node.Child0);
                    nodeCodeStack.AddLast(node.Child1);
                }
            }
            #endregion

            HuffTreeNode[] nodeArray = new HuffTreeNode[0x1FF]; // this array does not contain the leaves themselves!
            while (leafStemQueue.Count > 0)
            {
                Insert(leafStemQueue.First.Value, nodeArray, 0x3F + 1);
                leafStemQueue.RemoveFirst();
            }

            // update the indices to ignore all gaps
            int nodeIndex = 0;
            for (int i = 0; i < nodeArray.Length; i++)
            {
                if (nodeArray[i] != null)
                {
                    nodeArray[i].index = nodeIndex++;
                }
            }

            // write the nodes in their given order. However when 'writing' a node, write the data of its children instead.
            // the root node is always the first node.
            byte rootData = 0;
            if (root.Child0.IsData)
            {
                rootData |= 0x80;
            }
            if (root.Child1.IsData)
            {
                rootData |= 0x40;
            }
            outstream.WriteByte(rootData); compressedLength++;

            for (int i = 0; i < nodeArray.Length; i++)
            {
                if (nodeArray[i] != null)
                {
                    // nodes in this array are never data!
                    HuffTreeNode node0 = nodeArray[i].Child0;
                    if (node0.IsData)
                    {
                        outstream.WriteByte(node0.Data);
                    }
                    else
                    {
                        int offset = node0.index - nodeArray[i].index - 1;
                        if (offset > 0x3F)
                        {
                            throw new Exception("Offset overflow!");
                        }
                        byte data = (byte)offset;
                        if (node0.Child0.IsData)
                        {
                            data |= 0x80;
                        }
                        if (node0.Child1.IsData)
                        {
                            data |= 0x40;
                        }
                        outstream.WriteByte(data);
                    }

                    HuffTreeNode node1 = nodeArray[i].Child1;
                    if (node1.IsData)
                    {
                        outstream.WriteByte(node1.Data);
                    }
                    else
                    {
                        int offset = node1.index - nodeArray[i].index - 1;
                        if (offset > 0x3F)
                        {
                            throw new Exception("Offset overflow!");
                        }
                        byte data = (byte)offset;
                        if (node0.Child0.IsData)
                        {
                            data |= 0x80;
                        }
                        if (node0.Child1.IsData)
                        {
                            data |= 0x40;
                        }
                        outstream.WriteByte(data);
                    }

                    compressedLength += 2;
                }
            }
            #endregion

            #region write the data

            // the codewords are stored in blocks of 32 bits
            uint datablock       = 0;
            byte bitsLeftToWrite = 32;

            for (int i = 0; i < inLength; i++)
            {
                byte         data = inputData[i];
                HuffTreeNode node = leaves[data];
                // the depth of the node is the length of the codeword required to encode the byte
                int    depth = node.Depth;
                bool[] path  = new bool[depth];
                for (int d = 0; d < depth; d++)
                {
                    path[depth - d - 1] = node.IsChild1;
                    node = node.Parent;
                }
                for (int d = 0; d < depth; d++)
                {
                    if (bitsLeftToWrite == 0)
                    {
                        outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                        compressedLength += 4;
                        datablock         = 0;
                        bitsLeftToWrite   = 32;
                    }
                    bitsLeftToWrite--;
                    if (path[d])
                    {
                        datablock |= (uint)(1 << bitsLeftToWrite);
                    }
                    // no need to OR the buffer with 0 if it is child0
                }
            }

            // write the partly filled data block as well
            if (bitsLeftToWrite != 32)
            {
                outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                compressedLength += 4;
            }

            #endregion

            return(compressedLength);
        }
Exemplo n.º 8
0
        /// <summary>
        /// Inserts the given node into the given array, in such a location that
        /// the offset to both of its children is at most the given maximum, and as large as possible.
        /// In order to do this, the contents of the array may be shifted to the right.
        /// </summary>
        /// <param name="node">The node to insert.</param>
        /// <param name="array">The array to insert the node in.</param>
        /// <param name="maxOffset">The maximum offset between parent and children.</param>
        private void Insert(HuffTreeNode node, HuffTreeNode[] array, int maxOffset)
        {
            // if the node has two data-children, insert it as far to the end as possible.
            if (node.Child0.IsData && node.Child1.IsData)
            {
                for (int i = array.Length - 1; i >= 0; i--)
                {
                    if (array[i] == null)
                    {
                        array[i]   = node;
                        node.index = i;
                        break;
                    }
                }
            }
            else
            {
                // if the node is not data, insert it as far left as possible.
                // we know that both children are already present.
                int offset = Math.Max(node.Child0.index - maxOffset, node.Child1.index - maxOffset);
                offset = Math.Max(0, offset);
                if (offset >= node.Child0.index || offset >= node.Child1.index)
                {
                    // it may be that the childen are too far apart, with lots of empty entries in-between.
                    // shift the bottom child right until the node fits in its left-most place for the top child.
                    // (there should be more than enough room in the array)
                    while (offset >= Math.Min(node.Child0.index, node.Child1.index))
                    {
                        ShiftRight(array, Math.Min(node.Child0.index, node.Child1.index), maxOffset);
                    }
                    while (array[offset] != null)
                    {
                        ShiftRight(array, offset, maxOffset);
                    }
                    array[offset] = node;
                    node.index    = offset;
                }
                else
                {
                    for (int i = offset; i < node.Child0.index && i < node.Child1.index; i++)
                    {
                        if (array[i] == null)
                        {
                            array[i]   = node;
                            node.index = i;
                            break;
                        }
                    }
                }
            }

            if (node.index < 0)
            {
                throw new Exception("Node could not be inserted!");
            }

            // if the insertion of this node means that the parent has both children inserted, insert the parent.
            if (node.Parent != null)
            {
                if ((node.Parent.Child0.index >= 0 || node.Parent.Child0.IsData) &&
                    (node.Parent.Child1.index >= 0 || node.Parent.Child1.IsData))
                {
                    Insert(node.Parent, array, maxOffset);
                }
            }
        }
Exemplo n.º 9
0
        public override long Decompress(Stream instream, long inLength, Stream outstream)
        {
            #region GBATEK format specification
            /*
                Data Header (32bit)
                    Bit0-3   Data size in bit units (normally 4 or 8)
                    Bit4-7   Compressed type (must be 2 for Huffman)
                    Bit8-31  24bit size of decompressed data in bytes
                Tree Size (8bit)
                    Bit0-7   Size of Tree Table/2-1 (ie. Offset to Compressed Bitstream)
                Tree Table (list of 8bit nodes, starting with the root node)
                    Root Node and Non-Data-Child Nodes are:
                    Bit0-5   Offset to next child node,
                            Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2
                            Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1
                    Bit6     Node1 End Flag (1=Next child node is data)
                    Bit7     Node0 End Flag (1=Next child node is data)
                    Data nodes are (when End Flag was set in parent node):
                    Bit0-7   Data (upper bits should be zero if Data Size is less than 8)
                Compressed Bitstream (stored in units of 32bits)
                    Bit0-31  Node Bits (Bit31=First Bit)  (0=Node0, 1=Node1)
            */
            #endregion

            long readBytes = 0;

            byte type = (byte)instream.ReadByte();
            BlockSize blockSize = BlockSize.FOURBIT;
            if (type != (byte)blockSize)
                blockSize = BlockSize.EIGHTBIT;
            if (type != (byte)blockSize)
                throw new InvalidDataException(String.Format(Main.Get_Traduction("S05"), type.ToString("X")));
            byte[] sizeBytes = new byte[3];
            instream.Read(sizeBytes, 0, 3);
            int decompressedSize = IOUtils.ToNDSu24(sizeBytes, 0);
            readBytes += 4;
            if (decompressedSize == 0)
            {
                sizeBytes = new byte[4];
                instream.Read(sizeBytes, 0, 4);
                decompressedSize = IOUtils.ToNDSs32(sizeBytes, 0);
                readBytes += 4;
            }

            #region Read the Huff-tree

            if (readBytes >= inLength)
                throw new NotEnoughDataException(0, decompressedSize);
            int treeSize = instream.ReadByte(); readBytes++;
            if (treeSize < 0)
                throw new InvalidDataException(Main.Get_Traduction("S06"));

            treeSize = (treeSize + 1) * 2;

            if (readBytes + treeSize >= inLength)
                throw new InvalidDataException(Main.Get_Traduction("S07"));

            long treeEnd = (instream.Position - 1) + treeSize;

            // the relative offset may be 4 more (when the initial decompressed size is 0), but
            // since it's relative that doesn't matter, especially when it only matters if
            // the given value is odd or even.
            HuffTreeNode rootNode = new HuffTreeNode(instream, false, 5, treeEnd);

            readBytes += treeSize;
            // re-position the stream after the tree (the stream is currently positioned after the root
            // node, which is located at the start of the tree definition)
            instream.Position = treeEnd;

            #endregion

            // the current u32 we are reading bits from.
            uint data = 0;
            // the amount of bits left to read from <data>
            byte bitsLeft = 0;

            // a cache used for writing when the block size is four bits
            int cachedByte = -1;

            // the current output size
            int currentSize = 0;
            HuffTreeNode currentNode = rootNode;
            byte[] buffer = new byte[4];

            while (currentSize < decompressedSize)
            {
                #region find the next reference to a data node
                while (!currentNode.IsData)
                {
                    // if there are no bits left to read in the data, get a new byte from the input
                    if (bitsLeft == 0)
                    {
                        if (readBytes >= inLength)
                            throw new NotEnoughDataException(currentSize, decompressedSize);
                        int nRead = instream.Read(buffer, 0, 4);
                        if (nRead < 4)
                            throw new StreamTooShortException();
                        readBytes += nRead;
                        data = IOUtils.ToNDSu32(buffer, 0);
                        bitsLeft = 32;
                    }
                    // get the next bit
                    bitsLeft--;
                    bool nextIsOne = (data & (1 << bitsLeft)) != 0;
                    // go to the next node, the direction of the child depending on the value of the current/next bit
                    currentNode = nextIsOne ? currentNode.Child1 : currentNode.Child0;
                }
                #endregion

                #region write the data in the current node (when possible)
                switch (blockSize)
                {
                    case BlockSize.EIGHTBIT:
                        {
                            // just copy the data if the block size is a full byte
                            outstream.WriteByte(currentNode.Data);
                            currentSize++;
                            break;
                        }
                    case BlockSize.FOURBIT:
                        {
                            // cache the first half of the data if the block size is a half byte
                            if (cachedByte < 0)
                            {
                                cachedByte = currentNode.Data << 4;
                            }
                            else
                            {
                                // if we already cached a half-byte, combine the two halves and write the full byte.
                                cachedByte |= currentNode.Data;
                                outstream.WriteByte((byte)cachedByte);
                                currentSize++;
                                // be sure to forget the two written half-bytes
                                cachedByte = -1;
                            }
                            break;
                        }
                    default:
                        throw new Exception(String.Format(Main.Get_Traduction("S08"), blockSize.ToString()));
                }
                #endregion

                outstream.Flush();

                // make sure to start over next round
                currentNode = rootNode;
            }

            // the data is 4-byte aligned. Although very unlikely in this case (compressed bit blocks
            // are always 4 bytes long, and the tree size is generally 4-byte aligned as well),
            // skip any padding due to alignment.
            if (readBytes % 4 != 0)
                readBytes += 4 - (readBytes % 4);

            if (readBytes < inLength)
            {
                throw new TooMuchInputException(readBytes, inLength);
            }

            return decompressedSize;
        }
Exemplo n.º 10
0
            /// <summary>
            /// Creates a new node in the Huffman tree.
            /// </summary>
            /// <param name="stream">The stream to read from. It is assumed that there is (at least)
            /// one more byte available to read.</param>
            /// <param name="isData">If this node is a data-node.</param>
            /// <param name="relOffset">The offset of this node in the source data, relative to the start
            /// of the compressed file.</param>
            /// <param name="maxStreamPos">The indicated end of the huffman tree. If the stream is past
            /// this position, the tree is invalid.</param>
            public HuffTreeNode(Stream stream, bool isData, long relOffset, long maxStreamPos)
            {
                /*
                 Tree Table (list of 8bit nodes, starting with the root node)
                    Root Node and Non-Data-Child Nodes are:
                    Bit0-5   Offset to next child node,
                            Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2
                            Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1
                    Bit6     Node1 End Flag (1=Next child node is data)
                    Bit7     Node0 End Flag (1=Next child node is data)
                    Data nodes are (when End Flag was set in parent node):
                    Bit0-7   Data (upper bits should be zero if Data Size is less than 8)
                 */

                if (stream.Position >= maxStreamPos)
                {
                    // this happens when part of the tree is unused.
                    this.isFilled = false;
                    return;
                }
                this.isFilled = true;
                int readData = stream.ReadByte();
                if (readData < 0)
                    throw new StreamTooShortException();
                this.data = (byte)readData;

                this.isData = isData;

                if (!this.isData)
                {
                    int offset = this.data & 0x3F;
                    bool zeroIsData = (this.data & 0x80) > 0;
                    bool oneIsData = (this.data & 0x40) > 0;

                    // off AND NOT 1 == off XOR (off AND 1)
                    long zeroRelOffset = (relOffset ^ (relOffset & 1)) + offset * 2 + 2;

                    long currStreamPos = stream.Position;
                    // position the stream right before the 0-node
                    stream.Position += (zeroRelOffset - relOffset) - 1;
                    // read the 0-node
                    this.child0 = new HuffTreeNode(stream, zeroIsData, zeroRelOffset, maxStreamPos);
                    this.child0.Parent = this;
                    // the 1-node is directly behind the 0-node
                    this.child1 = new HuffTreeNode(stream, oneIsData, zeroRelOffset + 1, maxStreamPos);
                    this.child1.Parent = this;

                    // reset the stream position to right behind this node's data
                    stream.Position = currStreamPos;
                }
            }
Exemplo n.º 11
0
        /// <summary>
        /// Applies Huffman compression with a datablock size of 8 bits.
        /// </summary>
        /// <param name="instream">The stream to compress.</param>
        /// <param name="inLength">The length of the input stream.</param>
        /// <param name="outstream">The stream to write the decompressed data to.</param>
        /// <returns>The size of the decompressed data.</returns>
        public override int Compress(Stream instream, long inLength, Stream outstream)
        {
            if (inLength > 0xFFFFFF)
                throw new InputTooLargeException();

            // cache the input, as we need to build a frequency table
            byte[] inputData = new byte[inLength];
            instream.Read(inputData, 0, (int)inLength);

            // build that frequency table.
            int[] frequencies = new int[0x100];
            for (int i = 0; i < inLength; i++)
                frequencies[inputData[i]]++;

            #region Build the Huffman tree

            SimpleReversedPrioQueue<int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
            SimpleReversedPrioQueue<int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
            int nodeCount = 0;
            // make all leaf nodes, and put them in the leaf queue. Also save them for later use.
            HuffTreeNode[] leaves = new HuffTreeNode[0x100];
            for (int i = 0; i < 0x100; i++)
            {
                // there is no need to store leaves that are not used
                if (frequencies[i] == 0)
                    continue;
                HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
                leaves[i] = node;
                leafQueue.Enqueue(frequencies[i], node);
                nodeCount++;
            }

            while (leafQueue.Count + nodeQueue.Count > 1)
            {
                // get the two nodes with the lowest priority.
                HuffTreeNode one = null, two = null;
                int onePrio, twoPrio;
                one = GetLowest(leafQueue, nodeQueue, out onePrio);
                two = GetLowest(leafQueue, nodeQueue, out twoPrio);

                // give those two a common parent, and put that node in the node queue
                HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
                nodeQueue.Enqueue(onePrio + twoPrio, newNode);
                nodeCount++;
            }
            int rootPrio;
            HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
            // set the depth of all nodes in the tree, such that we know for each leaf how long
            // its codeword is.
            root.Depth = 0;

            #endregion

            // now that we have a tree, we can write that tree and follow with the data.

            // write the compression header first
            outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
            outstream.WriteByte((byte)(inLength & 0xFF));
            outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
            outstream.WriteByte((byte)((inLength >> 16) & 0xFF));

            int compressedLength = 4;

            #region write the tree

            outstream.WriteByte((byte)((nodeCount - 1) / 2));
            compressedLength++;

            // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the size of each sub-tree.
            // NO! BF results in an ordering that may overflow the offset field.
            
            // find the BF order of all nodes that have two leaves as children. We're going to insert them in an array in reverse BF order,
            // inserting the parent whenever both children have been inserted.
            
            LinkedList<HuffTreeNode> leafStemQueue = new LinkedList<HuffTreeNode>();

            #region fill the leaf queue; first->last will be reverse BF
            LinkedList<HuffTreeNode> nodeCodeStack = new LinkedList<HuffTreeNode>();
            nodeCodeStack.AddLast(root);
            while (nodeCodeStack.Count > 0)
            {
                HuffTreeNode node = nodeCodeStack.First.Value;
                nodeCodeStack.RemoveFirst();
                if (node.IsData)
                    continue;
                if (node.Child0.IsData && node.Child1.IsData)
                {
                    leafStemQueue.AddFirst(node);
                }
                else
                {
                    nodeCodeStack.AddLast(node.Child0);
                    nodeCodeStack.AddLast(node.Child1);
                }

            }
            #endregion

            HuffTreeNode[] nodeArray = new HuffTreeNode[0x1FF]; // this array does not contain the leaves themselves!
            while (leafStemQueue.Count > 0)
            {
                Insert(leafStemQueue.First.Value, nodeArray, 0x3F + 1);
                leafStemQueue.RemoveFirst();
            }

            // update the indices to ignore all gaps
            int nodeIndex = 0;
            for (int i = 0; i < nodeArray.Length; i++)
            {
                if (nodeArray[i] != null)
                    nodeArray[i].index = nodeIndex++;
            }

            // write the nodes in their given order. However when 'writing' a node, write the data of its children instead.
            // the root node is always the first node.
            byte rootData = 0;
            if (root.Child0.IsData)
                rootData |= 0x80;
            if (root.Child1.IsData)
                rootData |= 0x40;
            outstream.WriteByte(rootData); compressedLength++;

            for (int i = 0; i < nodeArray.Length; i++)
            {
                if (nodeArray[i] != null)
                {
                    // nodes in this array are never data!
                    HuffTreeNode node0 = nodeArray[i].Child0;
                    if (node0.IsData)
                        outstream.WriteByte(node0.Data);
                    else
                    {
                        int offset = node0.index - nodeArray[i].index - 1;
                        if (offset > 0x3F)
                            throw new Exception("Offset overflow!");
                        byte data = (byte)offset;
                        if (node0.Child0.IsData)
                            data |= 0x80;
                        if (node0.Child1.IsData)
                            data |= 0x40;
                        outstream.WriteByte(data);
                    }

                    HuffTreeNode node1 = nodeArray[i].Child1;
                    if (node1.IsData)
                        outstream.WriteByte(node1.Data);
                    else
                    {
                        int offset = node1.index - nodeArray[i].index - 1;
                        if (offset > 0x3F)
                            throw new Exception("Offset overflow!");
                        byte data = (byte)offset;
                        if (node0.Child0.IsData)
                            data |= 0x80;
                        if (node0.Child1.IsData)
                            data |= 0x40;
                        outstream.WriteByte(data);
                    }

                    compressedLength += 2;
                }
            }
            #endregion

            #region write the data

            // the codewords are stored in blocks of 32 bits
            uint datablock = 0;
            byte bitsLeftToWrite = 32;

            for (int i = 0; i < inLength; i++)
            {
                byte data = inputData[i];
                HuffTreeNode node = leaves[data];
                // the depth of the node is the length of the codeword required to encode the byte
                int depth = node.Depth;
                bool[] path = new bool[depth];
                for (int d = 0; d < depth; d++)
                {
                    path[depth - d - 1] = node.IsChild1;
                    node = node.Parent;
                }
                for (int d = 0; d < depth; d++)
                {
                    if (bitsLeftToWrite == 0)
                    {
                        outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                        compressedLength += 4;
                        datablock = 0;
                        bitsLeftToWrite = 32;
                    }
                    bitsLeftToWrite--;
                    if (path[d])
                        datablock |= (uint)(1 << bitsLeftToWrite);
                    // no need to OR the buffer with 0 if it is child0
                }
            }

            // write the partly filled data block as well
            if (bitsLeftToWrite != 32)
            {
                outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                compressedLength += 4;
            }

            #endregion

            return compressedLength;
        }
Exemplo n.º 12
0
        public static byte[] Huffman_Decompress(byte[] b, byte atype)
        {
            HUFF_STREAM instream  = new HUFF_STREAM(b);
            long        ReadBytes = 0;

            byte type = (byte)instream.ReadByte();

            type = atype;
            if (type != 0x28 && type != 0x24)
            {
                return(b);
            }
            int decompressedSize = instream.readThree();

            ReadBytes += 4;
            if (decompressedSize == 0)
            {
                instream.p      -= 3;
                decompressedSize = instream.ReadInt32();
                ReadBytes       += 4;
            }

            List <byte> o = new List <byte>();

            int treeSize = instream.ReadByte(); ReadBytes++;

            treeSize = (treeSize + 1) * 2;

            long treeEnd = (instream.p - 1) + treeSize;

            // the relative offset may be 4 more (when the initial decompressed size is 0), but
            // since it's relative that doesn't matter, especially when it only matters if
            // the given value is odd or even.
            HuffTreeNode rootNode = new HuffTreeNode(instream, false, 5, treeEnd);

            ReadBytes += treeSize;
            // re-position the stream after the tree (the stream is currently positioned after the root
            // node, which is located at the start of the tree definition)
            instream.p = (int)treeEnd;

            // the current u32 we are reading bits from.
            int data = 0;
            // the amount of bits left to read from <data>
            byte bitsLeft = 0;

            // a cache used for writing when the block size is four bits
            int cachedByte = -1;

            // the current output size
            HuffTreeNode currentNode = rootNode;

            while (instream.hasBytes())
            {
                while (!currentNode.isData)
                {
                    // if there are no bits left to read in the data, get a new byte from the input
                    if (bitsLeft == 0)
                    {
                        ReadBytes += 4;
                        data       = instream.ReadInt32();
                        bitsLeft   = 32;
                    }
                    // get the next bit
                    bitsLeft--;
                    bool nextIsOne = (data & (1 << bitsLeft)) != 0;
                    // go to the next node, the direction of the child depending on the value of the current/next bit
                    currentNode = nextIsOne ? currentNode.child1 : currentNode.child0;
                }

                switch (type)
                {
                case 0x28:
                {
                    // just copy the data if the block size is a full byte
                    //                        outstream.WriteByte(currentNode.Data);
                    o.Add(currentNode.data);
                    break;
                }

                case 0x24:
                {
                    // cache the first half of the data if the block size is a half byte
                    if (cachedByte < 0)
                    {
                        cachedByte = currentNode.data;
                    }
                    else
                    {
                        cachedByte |= currentNode.data << 4;
                        o.Add((byte)cachedByte);
                        cachedByte = -1;
                    }
                    break;
                }
                }

                currentNode = rootNode;
            }

            if (ReadBytes % 4 != 0)
            {
                ReadBytes += 4 - (ReadBytes % 4);
            }


            return(o.ToArray());
        }
Exemplo n.º 13
0
        public override byte[] Decompress(byte[] buffer, int offset)
        {
            BinaryReader br = new BinaryReader(new MemoryStream(buffer, offset, buffer.Length - offset));

            byte firstByte = br.ReadByte();
            int  dataSize  = firstByte & 0x0F;

            if (dataSize != 8 && dataSize != 4)
            {
                throw new Exception(string.Format("Unhandled Huffman dataSize {0:x}", dataSize));
            }

            int decomp_size = 0;

            for (int i = 0; i < 3; i++)
            {
                decomp_size |= br.ReadByte() << (i * 8);
            }

            byte treeSize = br.ReadByte();

            HuffTreeNode.maxInpos = 4 + (treeSize + 1) * 2;

            HuffTreeNode rootNode = new HuffTreeNode();

            rootNode.parseData(br);

            br.BaseStream.Position = 4 + (treeSize + 1) * 2;
            uint[] indata = new uint[(br.BaseStream.Length - br.BaseStream.Position) / 4];
            for (int i = 0; i < indata.Length; i++)
            {
                indata[i] = br.ReadUInt32();
            }

            long curr_size = 0;

            decomp_size *= dataSize == 8 ? 1 : 2;
            byte[] outdata = new byte[decomp_size];

            int               idx     = -1;
            string            codestr = "";
            LinkedList <byte> code    = new LinkedList <byte>();
            int               value;

            while (curr_size < decomp_size)
            {
                try
                {
                    string newstr = uint_to_bits(indata[++idx]);
                    codestr += newstr;
                }
                catch (IndexOutOfRangeException e)
                {
                    throw new IndexOutOfRangeException("not enough data.", e);
                }
                while (codestr.Length > 0)
                {
                    code.AddFirst(byte.Parse(codestr[0] + ""));
                    codestr = codestr.Remove(0, 1);
                    if (rootNode.getValue(code.Last, out value))
                    {
                        try
                        {
                            outdata[curr_size++] = (byte)value;
                        }
                        catch (IndexOutOfRangeException ex)
                        {
                            if (code.First.Value != 0)
                            {
                                throw ex;
                            }
                        }
                        code.Clear();
                    }
                }
            }

            br.Close();

            byte[] realout;
            if (dataSize == 4)
            {
                realout = new byte[decomp_size / 2];
                for (int i = 0; i < decomp_size / 2; i++)
                {
                    if ((outdata[i * 2] & 0xF0) > 0 || (outdata[i * 2 + 1] & 0xF0) > 0)
                    {
                        throw new Exception("first 4 bits of data should be 0 if dataSize = 4");
                    }
                    realout[i] = (byte)((outdata[i * 2] << 4) | outdata[i * 2 + 1]);
                }
            }
            else
            {
                realout = outdata;
            }

            return(realout);
        }
Exemplo n.º 14
0
        /// <summary>
        /// Inserts the given node into the given array, in such a location that
        /// the offset to both of its children is at most the given maximum, and as large as possible.
        /// In order to do this, the contents of the array may be shifted to the right.
        /// </summary>
        /// <param name="node">The node to insert.</param>
        /// <param name="array">The array to insert the node in.</param>
        /// <param name="maxOffset">The maximum offset between parent and children.</param>
        private void Insert(HuffTreeNode node, HuffTreeNode[] array, int maxOffset)
        {
            // if the node has two data-children, insert it as far to the end as possible.
            if (node.Child0.IsData && node.Child1.IsData)
            {
                for (int i = array.Length - 1; i >= 0; i--)
                {
                    if (array[i] == null)
                    {
                        array[i] = node;
                        node.index = i;
                        break;
                    }
                }
            }
            else
            {
                // if the node is not data, insert it as far left as possible.
                // we know that both children are already present.
                int offset = Math.Max(node.Child0.index - maxOffset, node.Child1.index - maxOffset);
                offset = Math.Max(0, offset);
                if (offset >= node.Child0.index || offset >= node.Child1.index)
                {
                    // it may be that the childen are too far apart, with lots of empty entries in-between.
                    // shift the bottom child right until the node fits in its left-most place for the top child.
                    // (there should be more than enough room in the array)
                    while (offset >= Math.Min(node.Child0.index, node.Child1.index))
                        ShiftRight(array, Math.Min(node.Child0.index, node.Child1.index), maxOffset);
                    while (array[offset] != null)
                        ShiftRight(array, offset, maxOffset);
                    array[offset] = node;
                    node.index = offset;
                }
                else
                {
                    for (int i = offset; i < node.Child0.index && i < node.Child1.index; i++)
                    {
                        if (array[i] == null)
                        {
                            array[i] = node;
                            node.index = i;
                            break;
                        }
                    }
                }
            }

            if (node.index < 0)
                throw new Exception("Node could not be inserted!");

            // if the insertion of this node means that the parent has both children inserted, insert the parent.
            if (node.Parent != null)
            {
                if ((node.Parent.Child0.index >= 0 || node.Parent.Child0.IsData)
                    && (node.Parent.Child1.index >= 0 || node.Parent.Child1.IsData))
                    Insert(node.Parent, array, maxOffset);
            }
        }
Exemplo n.º 15
0
        /// <summary>
        /// Applies Huffman compression with a datablock size of 8 bits.
        /// </summary>
        /// <param name="instream">The stream to compress.</param>
        /// <param name="inLength">The length of the input stream.</param>
        /// <param name="outstream">The stream to write the decompressed data to.</param>
        /// <returns>The size of the decompressed data.</returns>
        private int Compress8(Stream instream, long inLength, Stream outstream)
        {
            if (inLength > 0xFFFFFF)
            {
                throw new InputTooLargeException();
            }

            // cache the input, as we need to build a frequency table
            byte[] inputData = new byte[inLength];
            instream.Read(inputData, 0, (int)inLength);

            // build that frequency table.
            int[] frequencies = new int[0x100];
            for (int i = 0; i < inLength; i++)
            {
                frequencies[inputData[i]]++;
            }

            #region Build the Huffman tree

            SimpleReversedPrioQueue <int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
            SimpleReversedPrioQueue <int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue <int, HuffTreeNode>();
            int nodeCount = 0;
            // make all leaf nodes, and put them in the leaf queue. Also save them for later use.
            HuffTreeNode[] leaves = new HuffTreeNode[0x100];
            for (int i = 0; i < 0x100; i++)
            {
                // there is no need to store leaves that are not used
                if (frequencies[i] == 0)
                {
                    continue;
                }
                HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
                leaves[i] = node;
                leafQueue.Enqueue(frequencies[i], node);
                nodeCount++;
            }

            while (leafQueue.Count + nodeQueue.Count > 1)
            {
                // get the two nodes with the lowest priority.
                HuffTreeNode one = null, two = null;
                int          onePrio, twoPrio;
                one = GetLowest(leafQueue, nodeQueue, out onePrio);
                two = GetLowest(leafQueue, nodeQueue, out twoPrio);

                // give those two a common parent, and put that node in the node queue
                HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
                nodeQueue.Enqueue(onePrio + twoPrio, newNode);
                nodeCount++;
            }
            int          rootPrio;
            HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
            // set the depth of all nodes in the tree, such that we know for each leaf how long
            // its codeword is.
            root.Depth = 0;

            #endregion

            // now that we have a tree, we can write that tree and follow with the data.

            // write the compression header first
            outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
            outstream.WriteByte((byte)(inLength & 0xFF));
            outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
            outstream.WriteByte((byte)((inLength >> 16) & 0xFF));

            int compressedLength = 4;

            #region write the tree

            outstream.WriteByte((byte)((nodeCount - 1) / 2));
            compressedLength++;

            // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the side of each sub-tree.
            LinkedList <HuffTreeNode> printQueue = new LinkedList <HuffTreeNode>();
            printQueue.AddLast(root);
            while (printQueue.Count > 0)
            {
                HuffTreeNode node = printQueue.First.Value;
                printQueue.RemoveFirst();
                if (node.IsData)
                {
                    outstream.WriteByte(node.Data);
                }
                else
                {
                    // bits 0-5: 'offset' = # nodes in queue left
                    // bit 6: node1 end flag
                    // bit 7: node0 end flag
                    byte data = (byte)(printQueue.Count / 2);
                    data = (byte)(data & 0x3F);
                    if (node.Child0.IsData)
                    {
                        data |= 0x80;
                    }
                    if (node.Child1.IsData)
                    {
                        data |= 0x40;
                    }
                    outstream.WriteByte(data);

                    printQueue.AddLast(node.Child0);
                    printQueue.AddLast(node.Child1);
                }
                compressedLength++;
            }

            #endregion

            #region write the data

            // the codewords are stored in blocks of 32 bits
            uint datablock       = 0;
            byte bitsLeftToWrite = 32;

            for (int i = 0; i < inLength; i++)
            {
                byte         data = inputData[i];
                HuffTreeNode node = leaves[data];
                // the depth of the node is the length of the codeword required to encode the byte
                int    depth = node.Depth;
                bool[] path  = new bool[depth];
                for (int d = 0; d < depth; d++)
                {
                    path[depth - d - 1] = node.IsChild1;
                    node = node.Parent;
                }
                for (int d = 0; d < depth; d++)
                {
                    if (bitsLeftToWrite == 0)
                    {
                        outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                        compressedLength += 4;
                        datablock         = 0;
                        bitsLeftToWrite   = 32;
                    }
                    bitsLeftToWrite--;
                    if (path[d])
                    {
                        datablock |= (uint)(1 << bitsLeftToWrite);
                    }
                    // no need to OR the buffer with 0 if it is child0
                }
            }

            // write the partly filled data block as well
            if (bitsLeftToWrite != 32)
            {
                outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                compressedLength += 4;
            }

            #endregion

            return(compressedLength);
        }
Exemplo n.º 16
0
        public override long Decompress(Stream instream, long inLength, Stream outstream)
        {
            #region GBATEK format specification

            /*
             *  Data Header (32bit)
             *      Bit0-3   Data size in bit units (normally 4 or 8)
             *      Bit4-7   Compressed type (must be 2 for Huffman)
             *      Bit8-31  24bit size of decompressed data in bytes
             *  Tree Size (8bit)
             *      Bit0-7   Size of Tree Table/2-1 (ie. Offset to Compressed Bitstream)
             *  Tree Table (list of 8bit nodes, starting with the root node)
             *      Root Node and Non-Data-Child Nodes are:
             *      Bit0-5   Offset to next child node,
             *              Next child node0 is at (CurrentAddr AND NOT 1)+Offset*2+2
             *              Next child node1 is at (CurrentAddr AND NOT 1)+Offset*2+2+1
             *      Bit6     Node1 End Flag (1=Next child node is data)
             *      Bit7     Node0 End Flag (1=Next child node is data)
             *      Data nodes are (when End Flag was set in parent node):
             *      Bit0-7   Data (upper bits should be zero if Data Size is less than 8)
             *  Compressed Bitstream (stored in units of 32bits)
             *      Bit0-31  Node Bits (Bit31=First Bit)  (0=Node0, 1=Node1)
             */
            #endregion

            long readBytes = 0;

            byte      type      = (byte)instream.ReadByte();
            BlockSize blockSize = BlockSize.FOURBIT;
            if (type != (byte)blockSize)
            {
                blockSize = BlockSize.EIGHTBIT;
            }
            if (type != (byte)blockSize)
            {
                throw new InvalidDataException(String.Format(Main.Get_Traduction("S05"), type.ToString("X")));
            }
            byte[] sizeBytes = new byte[3];
            instream.Read(sizeBytes, 0, 3);
            int decompressedSize = IOUtils.ToNDSu24(sizeBytes, 0);
            readBytes += 4;
            if (decompressedSize == 0)
            {
                sizeBytes = new byte[4];
                instream.Read(sizeBytes, 0, 4);
                decompressedSize = IOUtils.ToNDSs32(sizeBytes, 0);
                readBytes       += 4;
            }

            #region Read the Huff-tree

            if (readBytes >= inLength)
            {
                throw new NotEnoughDataException(0, decompressedSize);
            }
            int treeSize = instream.ReadByte(); readBytes++;
            if (treeSize < 0)
            {
                throw new InvalidDataException(Main.Get_Traduction("S06"));
            }

            treeSize = (treeSize + 1) * 2;

            if (readBytes + treeSize >= inLength)
            {
                throw new InvalidDataException(Main.Get_Traduction("S07"));
            }

            long treeEnd = (instream.Position - 1) + treeSize;

            // the relative offset may be 4 more (when the initial decompressed size is 0), but
            // since it's relative that doesn't matter, especially when it only matters if
            // the given value is odd or even.
            HuffTreeNode rootNode = new HuffTreeNode(instream, false, 5, treeEnd);

            readBytes += treeSize;
            // re-position the stream after the tree (the stream is currently positioned after the root
            // node, which is located at the start of the tree definition)
            instream.Position = treeEnd;

            #endregion

            // the current u32 we are reading bits from.
            uint data = 0;
            // the amount of bits left to read from <data>
            byte bitsLeft = 0;

            // a cache used for writing when the block size is four bits
            int cachedByte = -1;

            // the current output size
            int          currentSize = 0;
            HuffTreeNode currentNode = rootNode;
            byte[]       buffer      = new byte[4];

            while (currentSize < decompressedSize)
            {
                #region find the next reference to a data node
                while (!currentNode.IsData)
                {
                    // if there are no bits left to read in the data, get a new byte from the input
                    if (bitsLeft == 0)
                    {
                        if (readBytes >= inLength)
                        {
                            throw new NotEnoughDataException(currentSize, decompressedSize);
                        }
                        int nRead = instream.Read(buffer, 0, 4);
                        if (nRead < 4)
                        {
                            throw new StreamTooShortException();
                        }
                        readBytes += nRead;
                        data       = IOUtils.ToNDSu32(buffer, 0);
                        bitsLeft   = 32;
                    }
                    // get the next bit
                    bitsLeft--;
                    bool nextIsOne = (data & (1 << bitsLeft)) != 0;
                    // go to the next node, the direction of the child depending on the value of the current/next bit
                    currentNode = nextIsOne ? currentNode.Child1 : currentNode.Child0;
                }
                #endregion

                #region write the data in the current node (when possible)
                switch (blockSize)
                {
                case BlockSize.EIGHTBIT:
                {
                    // just copy the data if the block size is a full byte
                    outstream.WriteByte(currentNode.Data);
                    currentSize++;
                    break;
                }

                case BlockSize.FOURBIT:
                {
                    // cache the first half of the data if the block size is a half byte
                    if (cachedByte < 0)
                    {
                        cachedByte = currentNode.Data << 4;
                    }
                    else
                    {
                        // if we already cached a half-byte, combine the two halves and write the full byte.
                        cachedByte |= currentNode.Data;
                        outstream.WriteByte((byte)cachedByte);
                        currentSize++;
                        // be sure to forget the two written half-bytes
                        cachedByte = -1;
                    }
                    break;
                }

                default:
                    throw new Exception(String.Format(Main.Get_Traduction("S08"), blockSize.ToString()));
                }
                #endregion

                outstream.Flush();

                // make sure to start over next round
                currentNode = rootNode;
            }

            // the data is 4-byte aligned. Although very unlikely in this case (compressed bit blocks
            // are always 4 bytes long, and the tree size is generally 4-byte aligned as well),
            // skip any padding due to alignment.
            if (readBytes % 4 != 0)
            {
                readBytes += 4 - (readBytes % 4);
            }

            if (readBytes < inLength)
            {
                throw new TooMuchInputException(readBytes, inLength);
            }

            return(decompressedSize);
        }
Exemplo n.º 17
0
        /// <summary>
        /// Applies Huffman compression with a datablock size of 8 bits.
        /// </summary>
        /// <param name="instream">The stream to compress.</param>
        /// <param name="inLength">The length of the input stream.</param>
        /// <param name="outstream">The stream to write the decompressed data to.</param>
        /// <returns>The size of the decompressed data.</returns>
        private int Compress8(Stream instream, long inLength, Stream outstream)
        {
            if (inLength > 0xFFFFFF)
                throw new InputTooLargeException();

            // cache the input, as we need to build a frequency table
            byte[] inputData = new byte[inLength];
            instream.Read(inputData, 0, (int)inLength);

            // build that frequency table.
            int[] frequencies = new int[0x100];
            for (int i = 0; i < inLength; i++)
                frequencies[inputData[i]]++;

            #region Build the Huffman tree

            SimpleReversedPrioQueue<int, HuffTreeNode> leafQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
            SimpleReversedPrioQueue<int, HuffTreeNode> nodeQueue = new SimpleReversedPrioQueue<int, HuffTreeNode>();
            int nodeCount = 0;
            // make all leaf nodes, and put them in the leaf queue. Also save them for later use.
            HuffTreeNode[] leaves = new HuffTreeNode[0x100];
            for (int i = 0; i < 0x100; i++)
            {
                // there is no need to store leaves that are not used
                if (frequencies[i] == 0)
                    continue;
                HuffTreeNode node = new HuffTreeNode((byte)i, true, null, null);
                leaves[i] = node;
                leafQueue.Enqueue(frequencies[i], node);
                nodeCount++;
            }

            while (leafQueue.Count + nodeQueue.Count > 1)
            {
                // get the two nodes with the lowest priority.
                HuffTreeNode one = null, two = null;
                int onePrio, twoPrio;
                one = GetLowest(leafQueue, nodeQueue, out onePrio);
                two = GetLowest(leafQueue, nodeQueue, out twoPrio);

                // give those two a common parent, and put that node in the node queue
                HuffTreeNode newNode = new HuffTreeNode(0, false, one, two);
                nodeQueue.Enqueue(onePrio + twoPrio, newNode);
                nodeCount++;
            }
            int rootPrio;
            HuffTreeNode root = nodeQueue.Dequeue(out rootPrio);
            // set the depth of all nodes in the tree, such that we know for each leaf how long
            // its codeword is.
            root.Depth = 0;

            #endregion

            // now that we have a tree, we can write that tree and follow with the data.

            // write the compression header first
            outstream.WriteByte((byte)BlockSize.EIGHTBIT); // this is block size 8 only
            outstream.WriteByte((byte)(inLength & 0xFF));
            outstream.WriteByte((byte)((inLength >> 8) & 0xFF));
            outstream.WriteByte((byte)((inLength >> 16) & 0xFF));

            int compressedLength = 4;

            #region write the tree

            outstream.WriteByte((byte)((nodeCount - 1) / 2));
            compressedLength++;

            // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the side of each sub-tree.
            LinkedList<HuffTreeNode> printQueue = new LinkedList<HuffTreeNode>();
            printQueue.AddLast(root);
            while (printQueue.Count > 0)
            {
                HuffTreeNode node = printQueue.First.Value;
                printQueue.RemoveFirst();
                if (node.IsData)
                {
                    outstream.WriteByte(node.Data);
                }
                else
                {
                    // bits 0-5: 'offset' = # nodes in queue left
                    // bit 6: node1 end flag
                    // bit 7: node0 end flag
                    byte data = (byte)(printQueue.Count / 2);
                    data = (byte)(data & 0x3F);
                    if (node.Child0.IsData)
                        data |= 0x80;
                    if (node.Child1.IsData)
                        data |= 0x40;
                    outstream.WriteByte(data);

                    printQueue.AddLast(node.Child0);
                    printQueue.AddLast(node.Child1);
                }
                compressedLength++;
            }

            #endregion

            #region write the data

            // the codewords are stored in blocks of 32 bits
            uint datablock = 0;
            byte bitsLeftToWrite = 32;

            for (int i = 0; i < inLength; i++)
            {
                byte data = inputData[i];
                HuffTreeNode node = leaves[data];
                // the depth of the node is the length of the codeword required to encode the byte
                int depth = node.Depth;
                bool[] path = new bool[depth];
                for (int d = 0; d < depth; d++)
                {
                    path[depth - d - 1] = node.IsChild1;
                    node = node.Parent;
                }
                for (int d = 0; d < depth; d++)
                {
                    if (bitsLeftToWrite == 0)
                    {
                        outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                        compressedLength += 4;
                        datablock = 0;
                        bitsLeftToWrite = 32;
                    }
                    bitsLeftToWrite--;
                    if (path[d])
                        datablock |= (uint)(1 << bitsLeftToWrite);
                    // no need to OR the buffer with 0 if it is child0
                }
            }

            // write the partly filled data block as well
            if (bitsLeftToWrite != 32)
            {
                outstream.Write(IOUtils.FromNDSu32(datablock), 0, 4);
                compressedLength += 4;
            }

            #endregion

            return compressedLength;
        }
Exemplo n.º 18
0
 /// <summary>
 /// Manually creates a new node for a huffman tree.
 /// </summary>
 /// <param name="data">The data for this node.</param>
 /// <param name="isData">If this node represents data.</param>
 /// <param name="child0">The child of this node on the 0 side.</param>
 /// <param name="child1">The child of this node on the 1 side.</param>
 public HuffTreeNode(byte data, bool isData, HuffTreeNode child0, HuffTreeNode child1)
 {
     this.data = data;
     this.isData = isData;
     this.child0 = child0;
     this.child1 = child1;
     this.isFilled = true;
     if (!isData)
     {
         this.child0.Parent = this;
         this.child1.Parent = this;
     }
 }
Exemplo n.º 19
0
 /// <summary>
 /// Shifts the node at the given index one to the right.
 /// If the distance between parent and child becomes too large due to this shift, the parent is shifted as well.
 /// </summary>
 /// <param name="array">The array to shift the node in.</param>
 /// <param name="idx">The index of the node to shift.</param>
 /// <param name="maxOffset">The maximum distance between parent and children.</param>
 private void ShiftRight(HuffTreeNode[] array, int idx, int maxOffset)
 {
     HuffTreeNode node = array[idx];
     if (array[idx + 1] != null)
         ShiftRight(array, idx + 1, maxOffset);
     if (node.Parent.index > 0 && node.index - maxOffset + 1 > node.Parent.index)
         ShiftRight(array, node.Parent.index, maxOffset);
     if (node != array[idx])
         return; // already done indirectly.
     array[idx + 1] = array[idx];
     array[idx] = null;
     node.index++;
 }