protected bool CanSendPackage(ref BitOutputStream output, IGameTime gameTime) { if (!outstandingPackages.Available(outSequence)) // running out here means we hit 64 packs without any acks from client... { // We have too many outstanding packages. We need the other end to send something to us, so we know he // is alive. This happens for example when we break the client in the debugger while the server is still // sending messages but potentially it could also happen in extreme cases of congestion or package loss. // We will try to send empty packages with low frequency to see if we can get the connection up and running again if (gameTime.FrameTime >= chokedTimeToNextPackage) { chokedTimeToNextPackage = gameTime.FrameTime + NetworkConfig.netChokeSendInterval.FloatValue; // Treat the last package as lost int chokedSequence; var info = outstandingPackages.TryGetByIndex(outSequence % outstandingPackages.Capacity, out chokedSequence); GameDebug.Assert(info != null); NotifyDelivered(chokedSequence, info, false); counters.chokedPackagesOut++; info.Reset(); outstandingPackages.Remove(chokedSequence); // Send empty package TPackageInfo emptyPackage; BeginSendPackage(ref output, out emptyPackage); CompleteSendPackage(emptyPackage, ref output); } return(false); } return(true); }
public void Close( ) { byte[] theInput = byteOut.ToArray( ); Console.WriteLine("Read " + theInput.Length + " bytes"); MemoryStream byteIn = new MemoryStream(theInput); CharCounter countObj = new CharCounter(byteIn); byteIn.Close( ); HuffmanTree codeTree = new HuffmanTree(countObj); codeTree.WriteEncodingTable(dout); BitOutputStream bout = new BitOutputStream(dout); for (int i = 0; i < theInput.Length; i++) { bout.WriteBits(codeTree.GetCode(theInput[i] & (0xff))); } bout.WriteBits(codeTree.GetCode(BitUtils.EOF)); bout.Close( ); byteOut.Close( ); }
public void BitStream_IntDelta_Random() { var buffer = new byte[1024 * 64]; var output = new BitOutputStream(buffer); var values = new int[1024]; var random = new Random(1032); long previous = 0; for (int i = 0; i < 1024; ++i) { values[i] = random.Next(int.MinValue, int.MaxValue); output.WriteIntDelta(values[i], previous); previous = values[i]; } output.Flush(); var input = new BitInputStream(buffer); previous = 0; for (int i = 0; i < 1024; ++i) { var value = input.ReadIntDelta(previous); Assert.AreEqual(values[i], value); previous = value; } }
protected bool CanSendPackage(ref BitOutputStream output) { // TODO (petera) track where else we clear from this if (!outstandingPackages.Available(outSequence)) // running out here means we hit 64 packs without any acks from client... { // We have too many outstanding packages. We need the other end to send something to us, so we know he // is alive. This happens for example when we break the client in the debugger while the server is still // sending messages but potentially it could also happen in extreme cases of congestion or package loss. // We will try to send empty packages with low frequency to see if we can get the connection up and running again if (chokedTimeToNextPackage.Update()) { // Treat the last package as lost int chokedSequence; var info = outstandingPackages.TryGetByIndex(outSequence % outstandingPackages.Capacity, out chokedSequence); GameDebug.Assert(info != null); NotifyDelivered(chokedSequence, info, false); counters.chokedPackagesOut++; info.Reset(); outstandingPackages.Remove(chokedSequence); // Send empty package TPackageInfo emptyPackage; BeginSendPackage(ref output, out emptyPackage); CompleteSendPackage(emptyPackage, ref output); } return(false); } // Check if are still limited by the bandwidth allowed for the connection return(timeToNextPackage.Update()); }
/// <summary> /// Return an int giving the node number of the root /// </summary> /// <param name="huffTree"></param> /// <param name="BOS"></param> /// <param name="NodesWritten"></param> /// <returns></returns> private int FindNodeNumber(BinaryTreeNode <byte> huffTree, BitOutputStream BOS, int NodesWritten) { if (huffTree.LeftChild == null && huffTree.RightChild == null) { BOS.WriteBits(0, 1); BOS.WriteBits(huffTree.Data, 8); NodesWritten++; return(NodesWritten); } else { // Recursively write the children int leftChild = FindNodeNumber(huffTree.LeftChild, BOS, NodesWritten); int rightChild = FindNodeNumber(huffTree.RightChild, BOS, leftChild); // Parent node will be +1 from right child NodesWritten = rightChild + 1; BOS.WriteBits(1, 1); BOS.WriteBits(leftChild, 9); BOS.WriteBits(rightChild, 9); return(NodesWritten); } }
public ArithmeticEncoder(BitOutputStream outStream) { const long fullRange = 1L << 32; _halfRange = fullRange >> 1; _quarterRange = _halfRange >> 1; _stateMask = fullRange - 1; _low = 0; _high = _stateMask; _output = outStream; _numUnderflow = 0; }
public void SendPackage <TOutputStream>() where TOutputStream : struct, NetworkCompression.IOutputStream { // We don't start sending updates before we have received at // least one content package from the server var rawOutputStream = new BitOutputStream(m_PackageBuffer); if (inSequence == 0 || !CanSendPackage(ref rawOutputStream)) { return; } // Only if there is anything to send // TODO (petera) should we send empty packages at a low frequency? if (sendClientConfig == false && commandSequence > 0 && commandSequence <= lastSentCommandSeq && eventsOut.Count == 0) { return; } ClientPackageInfo info; BeginSendPackage(ref rawOutputStream, out info); int endOfHeaderPos = rawOutputStream.Align(); var output = default(TOutputStream); // new TOutputStream(); due to bug new generate garbage here output.Initialize(NetworkCompressionModel.DefaultModel, m_PackageBuffer, endOfHeaderPos, null); if (sendClientConfig) { WriteClientConfig(ref output); } if (commandSequence > 0) { lastSentCommandSeq = commandSequence; WriteCommands(info, ref output); } WriteEvents(info, ref output); int compressedSize = output.Flush(); rawOutputStream.SkipBytes(compressedSize); CompleteSendPackage(info, ref rawOutputStream); }
protected int CompleteSendPackage(TPackageInfo info, ref BitOutputStream output) { info.SentTime = NetworkUtils.stopwatch.ElapsedMilliseconds; info.Content = (NetworkMessage)m_PackageBuffer[0]; int packageSize = output.Flush(); byte[] data = new byte[packageSize]; NetworkUtils.MemCopy(m_PackageBuffer, 0, data, 0, packageSize); counters.bytesOut += data.Length; Transport.SendData(ConnectionId, TransportEvent.Type.Data, data); counters.packagesOut++; ++outSequence; return(packageSize); }
public void BeginSendPackage(ref BitOutputStream output, out TPackageInfo info) { output.WriteBits(0, 8); // Package content flags (will set later as we add messages) output.WriteBits(Sequence.ToUInt16(outSequence), 16); output.WriteBits(Sequence.ToUInt16(inSequence), 16); output.WriteBits(inSequenceAckMask, 16); // Send rtt info every 3th package. We calculate the RTT as the time from sending the package // and receiving the ack for the package minus the time the package spent on the server if (outSequence % 3 == 0) { var now = NetworkUtils.stopwatch.ElapsedMilliseconds; var timeOnServer = (byte)Math.Min(now - inSequenceTime, 255); output.WriteBits(timeOnServer, 8); } info = outstandingPackages.Acquire(outSequence); }
/// <summary> /// Event handler for compression button /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void uxCompressButton_Click(object sender, EventArgs e) { try { if (uxOpenFileDialog1.ShowDialog() == DialogResult.OK) { string openFile = uxOpenFileDialog1.FileName; long numBytes; long[] ByteArray = BuildFrequencyTable(openFile, out numBytes); MinPriorityQueue <long, BinaryTreeNode <byte> > q = BuildHuffmanTreeLeaves(ByteArray); int numNodes = q.Count * 2 - 1; BinaryTreeNode <byte> huffTree = BuildHuffmanTree(q); long[] bitPaths = new long[256]; int[] pathLengths = new int[256]; GetVariableWidthEncoding(huffTree, 0, 0, bitPaths, pathLengths); string saveFile = null; if (uxSaveFileDialog1.ShowDialog() == DialogResult.OK) { saveFile = uxSaveFileDialog1.FileName; using (BitOutputStream BOS = new BitOutputStream(saveFile)) { // Writes first 63 bits, which describes length of file BOS.WriteBits(numBytes, 63); // Writes 9 bits, number of nodes BOS.WriteBits(numNodes, 9); // Post order tree traversal to assign node numbers int rootNumber = FindNodeNumber(huffTree, BOS, -1); CompressFile(openFile, BOS, bitPaths, pathLengths); } MessageBox.Show("Compressed File Written"); } } } catch (Exception ex) { MessageBox.Show(ex.ToString()); } }
public void BitStream_AlignAndByteArray() { var random = new Random(1293); var numbers = new int[1024]; var payload = new byte[32]; var payloadCompare = new byte[32]; random.NextBytes(payload); var buffer = new byte[1024 * 1024]; for (int runs = 0; runs < 1; ++runs) { for (int i = 0; i < 1024; ++i) { numbers[i] = random.Next(1, 33); } var output = new BitOutputStream(buffer); for (int i = 0; i < 1024; ++i) { output.WriteBits((uint)numbers[i], numbers[i]); if (i % 3 == 0) { output.WriteBytes(payload, 0, numbers[i]); } } var input = new BitInputStream(buffer); for (int i = 0; i < 1024; ++i) { var value = input.ReadBits(numbers[i]); Assert.AreEqual((uint)numbers[i], value); if (i % 3 == 0) { input.ReadBytes(payloadCompare, 0, numbers[i]); Assert.AreEqual(0, NetworkUtils.MemCmp(payload, 0, payloadCompare, 0, numbers[i])); } } } }
/// <summary> /// Method to compress the file /// </summary> /// <param name="fn"></param> /// <param name="BOS"></param> /// <param name="VarWidEnc"></param> /// <param name="lengths"></param> private void CompressFile(string fn, BitOutputStream BOS, long[] VarWidEnc, int[] lengths) { using (FileStream fs = new FileStream(fn, FileMode.Open, FileAccess.Read)) { // Read and append to Output stream int k; while ((k = fs.ReadByte()) != -1) { if (lengths[k] == 0) { return; } else { BOS.WriteBits(VarWidEnc[k], lengths[k]); } } } }
public static byte[] compressFloatsArray(float[] data, float precision) { float recPrecision = 1.0f / precision; int size = data.Length; int maxIndex = size > 0 ? floatToInt(data[0], recPrecision, precision) : 0; int minIndex = maxIndex; for (int i = 0; i < size; i++) { int value = floatToInt(data[i], recPrecision, precision); maxIndex = value < maxIndex ? maxIndex : value; minIndex = value > minIndex ? minIndex : value; } int bitSize = CPShortArrayData.getBitSize(maxIndex - minIndex); /* 7 bytes * 3 for minIndex * 1 for bitSize * 3 for size * 1 for rounding ((size * bitSize) >> 3) */ int bytesSize = ((size * bitSize) >> 3) + 8; BitOutputStream bitOutputStream = new BitOutputStream(bytesSize); bitOutputStream.WriteBits(24, minIndex); bitOutputStream.WriteBits(8, bitSize); bitOutputStream.WriteBits(24, size); for (int i = 0; i < size; i++) { int value = floatToInt(data[i], recPrecision, precision); bitOutputStream.WriteBits(bitSize, value - minIndex); } return(bitOutputStream.GetData()); }
public void BitStream_UIntPacked_RandomUInt() { var buffer = new byte[1024 * 64]; var output = new BitOutputStream(buffer); var values = new uint[1024]; var random = new Random(1032); for (int i = 0; i < 1024; ++i) { values[i] = (uint)random.Next(int.MaxValue); output.WriteUIntPacked(values[i]); } output.Flush(); var input = new BitInputStream(buffer); for (int i = 0; i < 1024; ++i) { var value = input.ReadUIntPacked(); Assert.AreEqual(values[i], value); } }
public void BitStream_Align() { var random = new Random(1293); var numbers = new int[1024]; var buffer = new byte[1024 * 64]; for (int runs = 0; runs < 1000; ++runs) { for (int i = 0; i < 1024; ++i) { numbers[i] = random.Next(1, 33); } var output = new BitOutputStream(buffer); for (int i = 0; i < 1024; ++i) { output.WriteBits((uint)numbers[i], numbers[i]); if (i % 3 == 0) { output.Align(); } } var input = new BitInputStream(buffer); for (int i = 0; i < 1024; ++i) { var value = input.ReadBits(numbers[i]); Assert.AreEqual((uint)numbers[i], value); if (i % 3 == 0) { input.Align(); } } } }
protected void BeginSendPackage(ref BitOutputStream output, out TPackageInfo info) { GameDebug.Assert(outstandingPackages.Available(outSequence), "NetworkConnection.BeginSendPackage : package info not available for sequence : {0}", outSequence); output.WriteBits(0, 8); // Package content flags (will set later as we add messages) output.WriteBits(Sequence.ToUInt16(outSequence), 16); output.WriteBits(Sequence.ToUInt16(inSequence), 16); output.WriteBits(inSequenceAckMask, 16); // Send rtt info every 3th package. We calculate the RTT as the time from sending the package // and receiving the ack for the package minus the time the package spent on the server // TODO should this be sent from client to server? if (outSequence % 3 == 0) { var now = NetworkUtils.stopwatch.ElapsedMilliseconds; // TOULF Is 255 enough? var timeOnServer = (byte)Math.Min(now - inSequenceTime, 255); output.WriteBits(timeOnServer, 8); } info = outstandingPackages.Acquire(outSequence); }
public void SendPackage() { var rawOutputStream = new BitOutputStream(m_PackageBuffer); // Distribute clients evenly according to their with snapshotInterval > 1 // TODO: This kind of assumes same update interval by all .... if ((_server.m_ServerSequence + ConnectionId) % snapshotInterval != 0) { return; } // Respect max bps rate cap if (Game.frameTime < nextOutPackageTime) { return; } ServerPackageInfo packageInfo; BeginSendPackage(ref rawOutputStream, out packageInfo); int endOfHeaderPos = rawOutputStream.Align(); var output = new RawOutputStream();// new TOutputStream(); Due to bug new generates garbage here output.Initialize(m_PackageBuffer, endOfHeaderPos); packageInfo.serverSequence = _server.m_ServerSequence; packageInfo.serverTime = _server.serverTime; // Server time (could be ticks or could be ms) // The ifs below are in essence the 'connection handshake' logic. if (!clientInfoAcked) { // Keep sending client info until it is acked WriteClientInfo(ref output); } else if (!mapAcked) { if (_server.m_MapInfo.serverInitSequence > 0) { // Keep sending map info until it is acked WriteMapInfo(ref output); } } else { // Send snapshot, buf only // if client has declared itself ready // if we have not already sent for this tick (because we need to be able to map a snapshot // sequence to a package sequence we cannot send the same snapshot multiple times). if (mapReady && _server.m_ServerSequence > snapshotServerLastWritten) { WriteSnapshot(ref output); } } int compressedSize = output.Flush(); rawOutputStream.SkipBytes(compressedSize); var messageSize = CompleteSendPackage(packageInfo, ref rawOutputStream); // Decide when next package can go out if (maxBPS > 0) { double timeLimitBPS = messageSize / maxBPS; if (timeLimitBPS > (float)snapshotInterval / Game.serverTickRate.FloatValue) { GameDebug.Log("SERVER: Choked by BPS sending " + messageSize); nextOutPackageTime = Game.frameTime + timeLimitBPS; } } CompleteSendPackage(packageInfo, ref rawOutputStream); }
/// <summary> /// Takes an in and out file, and configures all other methods to build the huffman tree and write it, as well as compress data. /// </summary> /// <param name="fs">Input file</param> /// <param name="outputStream">Output File</param> private void CompressStream(FileStream fs, BitOutputStream outputStream) { _encodingLength = new int[256]; _encodings = new long[256]; int numberOfNodes = 0; int inputLength; long[] frequency = GetFrequency(fs, out inputLength); outputStream.WriteBits(inputLength, 63); if(inputLength != 0) { BinaryTreeNode<byte> huff = BuildHuffmanTree(frequency, out numberOfNodes);/* TreeForm t = new TreeForm(huff, 100); t.Show();*/ ComputeEncodings(huff, 0, 0L); outputStream.WriteBits((2*numberOfNodes) - 1, 9); WriteHuffmanTree(huff, 0, outputStream); fs.Seek(0, SeekOrigin.Begin); WriteEncodings(fs, outputStream); } }
/// <summary> /// Event handler for compression button /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void uxCompressButton_Click(object sender, EventArgs e) { try { if (uxOpenFileDialog1.ShowDialog() == DialogResult.OK) { string openFile = uxOpenFileDialog1.FileName; long numBytes; long[] ByteArray = BuildFrequencyTable(openFile, out numBytes); MinPriorityQueue<long, BinaryTreeNode<byte>> q = BuildHuffmanTreeLeaves(ByteArray); int numNodes = q.Count * 2 - 1; BinaryTreeNode<byte> huffTree = BuildHuffmanTree(q); long[] bitPaths = new long[256]; int[] pathLengths = new int[256]; GetVariableWidthEncoding(huffTree,0,0,bitPaths,pathLengths); string saveFile = null; if (uxSaveFileDialog1.ShowDialog() == DialogResult.OK) { saveFile = uxSaveFileDialog1.FileName; using (BitOutputStream BOS = new BitOutputStream(saveFile)) { // Writes first 63 bits, which describes length of file BOS.WriteBits(numBytes, 63); // Writes 9 bits, number of nodes BOS.WriteBits(numNodes, 9); // Post order tree traversal to assign node numbers int rootNumber = FindNodeNumber(huffTree, BOS, -1); CompressFile(openFile, BOS, bitPaths, pathLengths); } MessageBox.Show("Compressed File Written"); } } } catch (Exception ex) { MessageBox.Show(ex.ToString()); } }
/// <summary> /// Return an int giving the node number of the root /// </summary> /// <param name="huffTree"></param> /// <param name="BOS"></param> /// <param name="NodesWritten"></param> /// <returns></returns> private int FindNodeNumber(BinaryTreeNode<byte> huffTree, BitOutputStream BOS, int NodesWritten) { if(huffTree.LeftChild == null && huffTree.RightChild == null) { BOS.WriteBits(0, 1); BOS.WriteBits(huffTree.Data, 8); NodesWritten++; return NodesWritten; } else { // Recursively write the children int leftChild = FindNodeNumber(huffTree.LeftChild, BOS, NodesWritten); int rightChild = FindNodeNumber(huffTree.RightChild, BOS, leftChild); // Parent node will be +1 from right child NodesWritten = rightChild + 1; BOS.WriteBits(1, 1); BOS.WriteBits(leftChild, 9); BOS.WriteBits(rightChild, 9); return NodesWritten; } }
/// <summary> /// Method to compress the file /// </summary> /// <param name="fn"></param> /// <param name="BOS"></param> /// <param name="VarWidEnc"></param> /// <param name="lengths"></param> private void CompressFile(string fn, BitOutputStream BOS, long[] VarWidEnc, int[] lengths) { using (FileStream fs = new FileStream(fn, FileMode.Open, FileAccess.Read)) { // Read and append to Output stream int k; while((k=fs.ReadByte()) != -1) { if(lengths[k] == 0) { return; } else { BOS.WriteBits(VarWidEnc[k], lengths[k]); } } } }
protected int CompleteSendPackage(TPackageInfo info, ref BitOutputStream output) { Profiler.BeginSample("NetworkConnection.CompleteSendPackage()"); info.sentTime = NetworkUtils.stopwatch.ElapsedMilliseconds; info.content = (NetworkMessage)m_PackageBuffer[0]; int packageSize = output.Flush(); GameDebug.Assert(packageSize < NetworkConfig.maxPackageSize, "packageSize < NetworkConfig.maxPackageSize"); if (debugSendStreamWriter != null) { debugSendStreamWriter.Write(m_PackageBuffer, 0, packageSize); debugSendStreamWriter.Write((UInt32)0xedededed); } if (packageSize > NetworkConfig.packageFragmentSize) { // Package is too big and needs to be sent as fragments var numFragments = packageSize / NetworkConfig.packageFragmentSize; //GameDebug.Log("FRAGMENTING: " + connectionId + ": " + packageSize + " (" + numFragments + ")"); var lastFragmentSize = packageSize % NetworkConfig.packageFragmentSize; if (lastFragmentSize != 0) { ++numFragments; } else { lastFragmentSize = NetworkConfig.packageFragmentSize; } for (var i = 0; i < numFragments; ++i) { var fragmentSize = i < numFragments - 1 ? NetworkConfig.packageFragmentSize : lastFragmentSize; var fragmentOutput = new BitOutputStream(m_FragmentBuffer); fragmentOutput.WriteBits((uint)NetworkMessage.FRAGMENT, 8); // Package fragment identifier fragmentOutput.WriteBits(Sequence.ToUInt16(outSequence), 16); fragmentOutput.WriteBits((uint)numFragments, 8); fragmentOutput.WriteBits((uint)i, 8); fragmentOutput.WriteBits((uint)fragmentSize, 16); fragmentOutput.WriteBytes(m_PackageBuffer, i * NetworkConfig.packageFragmentSize, fragmentSize); int fragmentPackageSize = fragmentOutput.Flush(); transport.SendData(connectionId, m_FragmentBuffer, fragmentPackageSize); counters.packagesOut++; counters.bytesOut += fragmentPackageSize; } counters.fragmentedPackagesOut++; } else { transport.SendData(connectionId, m_PackageBuffer, packageSize); counters.packagesOut++; counters.bytesOut += packageSize; } ++outSequence; Profiler.EndSample(); return(packageSize); }
/// <summary> /// Handles a Compress button click. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void uxCompressFile_Click(object sender, EventArgs e) { if (uxOpenFileDialog.ShowDialog() == DialogResult.OK) { if (uxSaveFileDialog.ShowDialog() == DialogResult.OK) { try { using (FileStream input = File.OpenRead(uxOpenFileDialog.FileName)) { using (BitOutputStream output = new BitOutputStream(uxSaveFileDialog.FileName)) { CompressStream(input, output); } } } catch (Exception ex) { DisplayError(ex); } } } }
/// <summary> /// Writes the encodings generated for each byte in FileStream to a outputstream. /// </summary> /// <param name="fs">FileStream of uncompressed data</param> /// <param name="outputStream">BitOutputStream of file to write compressed data to</param> private void WriteEncodings(FileStream fs, BitOutputStream outputStream) { int b = fs.ReadByte(); while (b != -1) { if (_encodingLength[b] > 0) { outputStream.WriteBits(_encodings[b], _encodingLength[b]); } b = fs.ReadByte(); } }
/// <summary> /// Writes the Huffman tree to the given OutputStream /// </summary> /// <param name="huff">Huffman tree to write</param> /// <param name="numberWritten">Amount of nodes written.</param> /// <param name="outputStream">BitOutputStream of file to write to.</param> /// <returns>An integer of the number of that node.</returns> private int WriteHuffmanTree(BinaryTreeNode<byte> huff, int numberWritten, BitOutputStream outputStream) { if (huff.LeftChild == null && huff.RightChild == null) { outputStream.WriteBits(1L, 1); outputStream.WriteBits(huff.RootValue, 8); return numberWritten; } else { int num1 = WriteHuffmanTree(huff.LeftChild, numberWritten++, outputStream); int num2 = WriteHuffmanTree(huff.RightChild, num1 + 1, outputStream); outputStream.WriteBits(0 << 1, 1); outputStream.WriteBits((byte)num1,9); outputStream.WriteBits((byte)num2,9); return num2 + 1; } }