Beispiel #1
0
        public void WritePackedUInt(uint value, int context)
        {
            int  bucket = NetworkCompressionUtils.CalculateBucket(value);
            uint offset = NetworkCompressionConstants.k_BucketOffsets[bucket];
            int  bits   = NetworkCompressionConstants.k_BucketSizes[bucket];

            WritePackedNibble((uint)bucket, context);
            if (bits > 0)
            {
                WriteRawBits(value - offset, bits);
            }
        }
        public void WritePackedUInt(uint value, int context)
        {
            if (m_Capture != null)
            {
                m_Capture.AddUInt(context, value);
            }

            int    bucket      = NetworkCompressionUtils.CalculateBucket(value);
            uint   offset      = NetworkCompressionConstants.k_BucketOffsets[bucket];
            int    bits        = NetworkCompressionConstants.k_BucketSizes[bucket];
            ushort encodeEntry = m_Model.encodeTable[context, bucket];

            WriteRawBitsInternal((uint)(encodeEntry >> 8), encodeEntry & 0xFF);
            WriteRawBitsInternal(value - offset, bits);
            FlushBits();
        }
Beispiel #3
0
        public NetworkCompressionModel(byte[] modelData)
        {
            if (modelData == null)
            {
                modelData = m_DefaultModelData;
            }

            int numContexts  = NetworkConfig.maxContexts;
            int alphabetSize = 16;

            byte[,] symbolLengths = new byte[numContexts, alphabetSize];

            int readOffset = 0;

            {
                // default model
                int defaultModelAlphabetSize = modelData[readOffset++];
                Debug.Assert(defaultModelAlphabetSize == alphabetSize);

                for (int i = 0; i < alphabetSize; i++)
                {
                    byte length = modelData[readOffset++];
                    for (int context = 0; context < numContexts; context++)
                    {
                        symbolLengths[context, i] = length;
                    }
                }

                // additional models
                int numModels = modelData[readOffset] | (modelData[readOffset + 1] << 8);
                readOffset += 2;
                for (int model = 0; model < numModels; model++)
                {
                    int context = modelData[readOffset] | (modelData[readOffset + 1] << 8);
                    readOffset += 2;

                    int modelAlphabetSize = modelData[readOffset++];
                    Debug.Assert(modelAlphabetSize == alphabetSize);
                    for (int i = 0; i < alphabetSize; i++)
                    {
                        byte length = modelData[readOffset++];
                        symbolLengths[context, i] = length;
                    }
                }
            }

            // generate tables
            encodeTable = new ushort[numContexts, alphabetSize];
            decodeTable = new ushort[numContexts, 1 << NetworkCompressionConstants.k_MaxHuffmanSymbolLength];

            var tmpSymbolLengths     = new byte[alphabetSize];
            var tmpSymbolDecodeTable = new ushort[1 << NetworkCompressionConstants.k_MaxHuffmanSymbolLength];
            var symbolCodes          = new byte[alphabetSize];

            for (int context = 0; context < numContexts; context++)
            {
                for (int i = 0; i < alphabetSize; i++)
                {
                    tmpSymbolLengths[i] = symbolLengths[context, i];
                }

                NetworkCompressionUtils.GenerateHuffmanCodes(symbolCodes, 0, tmpSymbolLengths, 0, alphabetSize, NetworkCompressionConstants.k_MaxHuffmanSymbolLength);
                NetworkCompressionUtils.GenerateHuffmanDecodeTable(tmpSymbolDecodeTable, 0, tmpSymbolLengths, symbolCodes, alphabetSize, NetworkCompressionConstants.k_MaxHuffmanSymbolLength);
                for (int i = 0; i < alphabetSize; i++)
                {
                    encodeTable[context, i] = (ushort)((symbolCodes[i] << 8) | symbolLengths[context, i]);
                }
                for (int i = 0; i < (1 << NetworkCompressionConstants.k_MaxHuffmanSymbolLength); i++)
                {
                    decodeTable[context, i] = tmpSymbolDecodeTable[i];
                }
            }
            this.modelData = modelData;
        }
Beispiel #4
0
        public byte[] AnalyzeAndGenerateModel()
        {
            int       alphabetSize = 16;
            var       model        = m_Model;
            int       numContexts  = uintData.Length;
            const int numBuckets   = NetworkCompressionConstants.k_NumBuckets;

            var stringWriter = new StringWriter();

            int[] histogram     = new int[alphabetSize];
            int[] safeHistogram = new int[alphabetSize];
            int[,] histogram2     = new int[alphabetSize, alphabetSize];
            int[,] safeHistogram2 = new int[alphabetSize, alphabetSize];

            Directory.CreateDirectory("capture");
            stringWriter.WriteLine("NetworkProfile:");
            var combinedSOAFile = System.IO.File.OpenWrite("capture/combined_soa.dat");

            List <byte> modelData = new List <byte>();

            modelData.Add(16);
            modelData.AddRange(new byte[] { 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6 });

            int numContextsOffset = modelData.Count;

            modelData.Add(0);   //num contexts
            modelData.Add(0);

            int           totalNumValues     = 0;
            int           gammaTotalCost     = 0;
            int           currentTotalCost   = 0;
            int           optimizedTotalCost = 0;
            double        entropyTotalCost   = 0.0;
            double        entropy2TotalCost  = 0.0;
            int           numUsedContexts    = 0;
            List <string> optimizedTrees     = new List <string>();

            for (int context = 0; context < numContexts; context++)
            {
                if (uintData[context].Count == 0 && nibbleData[context].Count == 0)
                {
                    continue;
                }


                bool isUInt = uintData[context].Count > 0;


                int numValues = 0;
                if (isUInt)
                {
                    Debug.Assert(nibbleData[context].Count == 0);
                    alphabetSize = numBuckets;
                    numValues    = uintData[context].Count;
                }
                else
                {
                    Debug.Assert(uintData[context].Count == 0);
                    Debug.Assert(nibbleData[context].Count > 0);
                    alphabetSize = 16;
                    numValues    = nibbleData[context].Count;
                }

                // build histograms
                for (int i = 0; i < numBuckets; i++)
                {
                    histogram[i] = 0;
                }

                for (int i = 0; i < alphabetSize; i++)
                {
                    for (int j = 0; j < alphabetSize; j++)
                    {
                        histogram2[i, j] = 0;
                    }
                }

                combinedSOAFile.WriteByte((byte)(numValues & 0xFF));
                combinedSOAFile.WriteByte((byte)((numValues >> 8) & 0xFF));
                combinedSOAFile.WriteByte((byte)((numValues >> 16) & 0xFF));
                combinedSOAFile.WriteByte((byte)((numValues >> 24) & 0xFF));

                var contextFile = System.IO.File.OpenWrite("capture/context" + context);

                int gammaCost  = 0;
                int prevSymbol = 0;
                for (int i = 0; i < numValues; i++)
                {
                    uint value;
                    if (isUInt)
                    {
                        value = uintData[context][i];
                        int bucket = NetworkCompressionUtils.CalculateBucket(value);
                        histogram[bucket]++;
                        histogram2[prevSymbol, bucket]++;
                        prevSymbol = bucket;
                    }
                    else
                    {
                        value = nibbleData[context][i];
                        histogram[value]++;
                        histogram2[prevSymbol, value]++;
                        prevSymbol = (int)value;
                    }

                    gammaCost += NetworkCompressionUtils.CalculateNumGammaBits(value);

                    combinedSOAFile.WriteByte((byte)(value & 0xFF));
                    combinedSOAFile.WriteByte((byte)((value >> 8) & 0xFF));
                    combinedSOAFile.WriteByte((byte)((value >> 16) & 0xFF));
                    combinedSOAFile.WriteByte((byte)((value >> 24) & 0xFF));

                    contextFile.WriteByte((byte)(value & 0xFF));
                    contextFile.WriteByte((byte)((value >> 8) & 0xFF));
                    contextFile.WriteByte((byte)((value >> 16) & 0xFF));
                    contextFile.WriteByte((byte)((value >> 24) & 0xFF));
                }
                contextFile.Close();

                // safe histogram where all values have at least one occurrence
                int safeNumValues = numValues;
                for (int i = 0; i < alphabetSize; i++)
                {
                    int n = histogram[i];
                    if (n == 0)
                    {
                        n = 1;
                        safeNumValues++;
                    }
                    safeHistogram[i] = n;
                }


                byte[] optimizedSymbolLengths = new byte[alphabetSize];
                NetworkCompressionUtils.GenerateLengthLimitedHuffmanCodeLengths(optimizedSymbolLengths, 0, safeHistogram, alphabetSize, NetworkCompressionConstants.k_MaxHuffmanSymbolLength);
                modelData.Add((byte)(context & 0xFF));
                modelData.Add((byte)(context >> 8));
                modelData.Add((byte)alphabetSize);
                for (int i = 0; i < alphabetSize; i++)
                {
                    modelData.Add((byte)optimizedSymbolLengths[i]);
                }

                int    currentCost   = 0;
                int    optimizedCost = 0;
                double entropyCost   = 0.0;

                for (int i = 0; i < alphabetSize; i++)
                {
                    int n = histogram[i];
                    if (n > 0)
                    {
                        int currentBitLength   = model.encodeTable[context, i] & 0xFF;
                        int optimizedBitLength = optimizedSymbolLengths[i] & 0xFF;
                        currentCost   += n * currentBitLength;
                        optimizedCost += n * optimizedBitLength;
                        double p = n / (double)safeNumValues;
                        entropyCost += n * -Math.Log(p, 2.0);
                        if (isUInt)
                        {
                            currentCost   += n * NetworkCompressionConstants.k_BucketSizes[i];
                            optimizedCost += n * NetworkCompressionConstants.k_BucketSizes[i];
                            entropyCost   += n * NetworkCompressionConstants.k_BucketSizes[i];
                        }
                    }
                }

                double entropy2Cost = 0.0;
                for (int i = 0; i < alphabetSize; i++)
                {
                    int total = 0;
                    for (int j = 0; j < alphabetSize; j++)
                    {
                        int n = histogram2[i, j];
                        if (n == 0)
                        {
                            n = 1;
                        }
                        safeHistogram2[i, j] = n;
                        total += n;
                    }

                    for (int j = 0; j < alphabetSize; j++)
                    {
                        int n = histogram2[i, j];
                        if (n > 0)
                        {
                            double p = n / (double)total;
                            entropy2Cost += n * -Math.Log(p, 2.0);
                            if (isUInt)
                            {
                                entropy2Cost += n * NetworkCompressionConstants.k_BucketSizes[j];
                            }
                        }
                    }
                }

                totalNumValues     += numValues;
                gammaTotalCost     += gammaCost;
                currentTotalCost   += currentCost;
                optimizedTotalCost += optimizedCost;
                entropyTotalCost   += entropyCost;
                entropy2TotalCost  += entropy2Cost;
                var    l          = new List <byte>(optimizedSymbolLengths);
                string symLengths = string.Join(":", l);
                stringWriter.WriteLine("{0,4}:   {1,8} {2,8:0.00} {3,8:0.00} {4,8:0.00} {5,8:0.00} {6,8:0.00}   {7}", context, numValues, gammaCost / 8.0f, currentCost / 8.0f, optimizedCost / 8.0f, entropyCost / 8.0, entropy2Cost / 8.0, symLengths);
                optimizedTrees.Add("" + string.Format("{0,10:000000}", currentCost - optimizedCost) + " " + symLengths);

                numUsedContexts++;
            }

            optimizedTrees.Sort();
            foreach (var l in optimizedTrees)
            {
                GameDebug.Log("  " + l);
            }

            stringWriter.WriteLine("Total: {0,8} {1,8:0.00} {2,8:0.00} {3,8:0.00} {4,8:0.00} {5,8:0.00}", totalNumValues, gammaTotalCost / 8.0f, currentTotalCost / 8.0f, optimizedTotalCost / 8.0f, entropyTotalCost / 8.0, entropy2TotalCost / 8.0);
            stringWriter.WriteLine("Num used contexts: {0}", numUsedContexts);
            GameDebug.Log(stringWriter.ToString());

            combinedSOAFile.Close();

            System.IO.File.WriteAllBytes("capture/combined_aos.dat", rawData.ToArray());


            modelData[numContextsOffset + 0] = (byte)(numUsedContexts & 0xFF);
            modelData[numContextsOffset + 1] = (byte)(numUsedContexts >> 8);

            return(modelData.ToArray());
        }