protected override void Parse(ref BitStreamReader bsr)
{
Cmd = bsr.ReadUInt();
uint byteSize = bsr.ReadUInt();
int indexBeforeData = bsr.CurrentBitIndex;
CommandNumber = bsr.ReadUIntIfExists();
TickCount = bsr.ReadUIntIfExists();
ViewAngleX = bsr.ReadFloatIfExists();
ViewAngleY = bsr.ReadFloatIfExists();
ViewAngleZ = bsr.ReadFloatIfExists();
SidewaysMovement = bsr.ReadFloatIfExists();
ForwardMovement = bsr.ReadFloatIfExists();
VerticalMovement = bsr.ReadFloatIfExists();
Buttons = (Buttons?)bsr.ReadUIntIfExists();
Impulse = bsr.ReadByteIfExists();
if (bsr.ReadBool())
{
WeaponSelect = bsr.ReadUInt(11);
WeaponSubtype = bsr.ReadUIntIfExists(6);
}
MouseDx = (short?)bsr.ReadUShortIfExists();
MouseDy = (short?)bsr.ReadUShortIfExists();
bsr.CurrentBitIndex = indexBeforeData + (int)(byteSize << 3);
}
protected override void Parse(ref BitStreamReader bsr)
{
ClassCount = bsr.ReadUShort();
CreateOnClient = bsr.ReadBool();
if (!CreateOnClient)
{
// if this ever gets used then it should update the mutable tables
string s = $"I haven't implemented {GetType().Name} to update the C_string tables.";
DemoRef.LogError(s);
Debug.WriteLine(s);
ServerClasses = new ServerClass[ClassCount];
for (int i = 0; i < ServerClasses.Length; i++)
{
ServerClasses[i] = new ServerClass(DemoRef, this);
ServerClasses[i].ParseStream(ref bsr);
// this is an assumption I make in the structure of all the entity stuff, very critical
if (i != ServerClasses[i].DataTableId)
{
throw new ConstraintException("server class ID does not match it's index in the list");
}
}
}
DemoRef.EntBaseLines ??= new EntityBaseLines(DemoRef, ClassCount);
}
protected override void Parse(ref BitStreamReader bsr)
{
PacketInfo = new CmdInfo[DemoInfo.MaxSplitscreenPlayers];
for (int i = 0; i < PacketInfo.Length; i++)
{
PacketInfo[i] = new CmdInfo(DemoRef);
PacketInfo[i].ParseStream(ref bsr);
}
InSequence = bsr.ReadUInt();
OutSequence = bsr.ReadUInt();
MessageStream = new MessageStream(DemoRef);
MessageStream.ParseStream(ref bsr);
// After we're doing with the packet, we can process all the messages.
// Most things should be processed during parsing, but any additional checks should be done here.
var netTickMessages = MessageStream.Where(tuple => tuple.messageType == MessageType.NetTick).ToList();
if (netTickMessages.Count > 1)
{
DemoRef.LogError("there's more than 2 net tick messages in this packet");
}
NetTick?tickInfo = (NetTick?)netTickMessages.FirstOrDefault().message;
if (tickInfo != null)
{
if (DemoRef.EntitySnapshot != null)
{
DemoRef.EntitySnapshot.EngineTick = tickInfo.EngineTick;
}
}
// todo fill prop handles with data here
TimingAdjustment.AdjustFromPacket(this);
}
protected override void Parse(ref BitStreamReader bsr)
{
RemoveUser = bsr.ReadBool();
uint dataLen = bsr.ReadUInt(11);
Data = bsr.SplitAndSkip(dataLen);
}
protected override void Parse(ref BitStreamReader bsr)
{
Info = new List <MapCompletedInfo>();
Span <byte> bytes = stackalloc byte[2 * MaxPortal2CoopBranches * MaxPortal2CoopLevelsPerBranch / 8];
bsr.ReadToSpan(bytes);
int current = 0;
int mask = 0x01;
for (int player = 0; player < 2; player++)
{
for (int branch = 0; branch < MaxPortal2CoopBranches; branch++)
{
for (int level = 0; level < MaxPortal2CoopLevelsPerBranch; level++)
{
if ((bytes[current] & mask) != 0)
{
Info.Add(new MapCompletedInfo {
Player = player, Branch = branch, Level = level
});
}
mask <<= 1;
if (mask >= 0x0100)
{
current++;
mask = 0x01;
}
}
}
}
}
public static UInt64 ReadUInt64(this BitStreamReader reader)
{
UInt32 left = reader.ReadUInt32();
UInt32 right = reader.ReadUInt32();
return(BitConverterX.UInt32ToUInt64(left, right));
}
public void Part1()
{
string data = IO.ReadFile(Path.Combine(Directory.GetCurrentDirectory(), "input1.txt"));
List <byte> packet = new List <byte>();
for (int i = 1; i < data.Length; i += 2)
{
string byteStr = string.Concat(data[i - 1], data[i]);
packet.Add(byte.Parse(byteStr, NumberStyles.HexNumber));
}
List <Packet> packets = new List <Packet>();
BitStreamReader reader = new BitStreamReader(packet.ToArray());
ulong sumVersion = 0;
Packet top = Packet.CreatePacket(reader);
Queue <Packet> queue = new Queue <Packet>();
queue.Enqueue(top);
while (queue.Count > 0)
{
Packet p = queue.Dequeue();
sumVersion += p.Version;
if (p is OperatorPacket op)
{
foreach (Packet inner in op.Packets)
{
queue.Enqueue(inner);
}
}
}
Assert.Equal((ulong)0, sumVersion);
}
public static unsafe byte[] Decompress(ref BitStreamReader bsr, int compSize)
{
uint type = bsr.ReadUInt();
int decompSize = bsr.ReadSInt();
switch (type)
{
case LZSS_ID:
Span <byte> dataIn = compSize < 100000 ? stackalloc byte[compSize] : new byte[compSize];
bsr.ReadToSpan(dataIn);
byte[] arrOut = new byte[decompSize];
fixed(byte *pDataIn = dataIn)
fixed(byte *pDataOut = arrOut)
DecompressLZSS(pDataIn, pDataOut);
return(arrOut);
case SNAPPY_ID:
Debug.Assert(false);
throw new NotImplementedException("snappy decompression not implemented");
default:
throw new InvalidOperationException("unknown compression method");
}
}
public object CreateNewObject(BitStreamReader reader, ObjectContract objectContract, DsdlProperty containerMember, DsdlProperty containerProperty, out bool createdFromNonDefaultCreator)
{
object newObject = null;
if (objectContract.DefaultCreator != null && !objectContract.DefaultCreatorNonPublic)
{
// use the default constructor if it is...
// public
// non-public and the user has change constructor handling settings
// non-public and there is no other creator
newObject = objectContract.DefaultCreator();
}
if (newObject == null)
{
if (!objectContract.IsInstantiable)
{
throw new SerializationException("Could not create an instance of type {0}. Type is an interface or abstract class and cannot be instantiated.".FormatWith(CultureInfo.InvariantCulture, objectContract.UnderlyingType));
}
throw new SerializationException("Unable to find a constructor to use for type {0}. A class should either have a default constructor, one constructor with arguments or a constructor marked with the JsonConstructor attribute.".FormatWith(CultureInfo.InvariantCulture, objectContract.UnderlyingType));
}
createdFromNonDefaultCreator = false;
return(newObject);
}
object CreateValueInternal(
BitStreamReader reader,
IContract contract,
DsdlProperty member,
ContainerContract containerContract,
DsdlProperty containerMember,
object existingValue,
DsdlType scheme,
Type objectType,
bool tailArrayOptimization = false)
{
switch (scheme)
{
case VoidDsdlType t:
ReadAlignment(reader, t);
return(null);
case PrimitiveDsdlType t:
var primitive = ReadPrimitiveType(reader, t);
return(EnsureType(reader, primitive, CultureInfo.InvariantCulture, contract, objectType));
case ArrayDsdlType t when t.IsStringLike && contract.UnderlyingType == typeof(string):
var list = CreateList(reader, StringContract, member, null, t, tailArrayOptimization) as IEnumerable <byte>;
return(_encoding.GetString(list.ToArray()));
case ArrayDsdlType t:
return(CreateList(reader, contract, member, existingValue, t, tailArrayOptimization));
case CompositeDsdlTypeBase t:
return(CreateObject(reader, contract, member, containerContract, containerMember, existingValue, objectType, t, tailArrayOptimization));
default:
throw new ArgumentOutOfRangeException(nameof(scheme));
}
}
public void VerifyCompression_LargeCorpus()
{
var stream = new MemoryStream();
var writer = new BitStreamWriter(stream, true);
var reader = new BitStreamReader(stream, true);
string input = TestResources.RFC5_Text;
var state = new TreeStateStore();
var compressor = new StaticHuffman <char>(CharacterFrequencies(input));
compressor.WriteTable(state.WriteSymbol, state.WriteUInt32);
foreach (char ch in input)
{
compressor.WriteCode(ch, writer.Write);
}
writer.Flush();
state.Reset();
var decompressor = new StaticHuffman <char>(state.ReadSymbol, state.ReadUInt32);
stream.Position = 0;
foreach (char ch in input)
{
Assert.AreEqual(ch, decompressor.GetSymbol(reader.ReadBoolean));
}
}
public void EightBitsRead()
{
var data = new byte[] { 0x3c, 0x4b, 0x1a, 0x54, 0x22, 0x10, 0xaf, 0x89, 0x97, 0x65 };
var stream = new MemoryStream(data);
var reader = new BitStreamReader(stream);
Assert.AreEqual(true, reader.CanRead);
Assert.AreEqual(true, reader.CanSeek);
Assert.AreEqual(false, reader.CanWrite);
Assert.AreEqual(80, reader.BitLength);
Assert.AreEqual(0, reader.BitPosition);
Assert.AreEqual(10, reader.Length);
Assert.AreEqual(0, reader.Position);
Assert.AreEqual(false, reader.ReadBoolean());
Assert.AreEqual(false, reader.ReadBoolean());
Assert.AreEqual(true, reader.ReadBoolean());
Assert.AreEqual(true, reader.ReadBoolean());
Assert.AreEqual(true, reader.ReadBoolean());
Assert.AreEqual(true, reader.ReadBoolean());
Assert.AreEqual(false, reader.ReadBoolean());
Assert.AreEqual(false, reader.ReadBoolean());
Assert.AreEqual(80, reader.BitLength);
Assert.AreEqual(8, reader.BitPosition);
Assert.AreEqual(10, reader.Length);
Assert.AreEqual(1, reader.Position);
}
protected override void Parse(ref BitStreamReader bsr)
{
Name = bsr.ReadNullTerminatedString();
ushort entryCount = bsr.ReadUShort();
if (entryCount > 0)
{
TableEntries = new List <StringTableEntry>(entryCount);
for (int i = 0; i < entryCount; i++)
{
var entry = new StringTableEntry(DemoRef, this);
TableEntries.Add(entry);
entry.ParseStream(ref bsr);
}
}
if (bsr.ReadBool())
{
ushort classCount = bsr.ReadUShort();
Classes = new List <StringTableClass>(classCount);
for (int i = 0; i < classCount; i++)
{
var @class = new StringTableClass(DemoRef);
Classes.Add(@class);
@class.ParseStream(ref bsr);
}
}
}
public void init(byte[] bs)
{
BitStreamReader sr = new BitStreamReader(bs);
SetHead(bs);
Protection = (byte)(bs[1] & 0x01);
if (Protection == 0) //ADTS有9byte
{
}
else //ADTS有7byte
{
}
/*
* 0: AAC Main
* 1: AAC LC (Low Complexity)
* 2: AAC SSR (Scalable Sample Rate)
* 3: AAC LTP (Long Term Prediction)
*/
//编码深度
Profile = (byte)((bs[2] & 0xC0) >> 6);
/*MPEG_4_Sampling_Frequency_Index 对应值
* 0x0 96000
* 0x1 88200
* 0x2 64000
* 0x3 48000
* 0x4 44100
* 0x5 32000
* 0x6 24000
* 0x7 22050
* 0x8 16000
* 0x9 2000
* 0xa 11025
* 0xb 8000
* 0xc reserved
* 0xd reserved
* 0xe reserved
* 0xf reserved
*/
//采样率
MPEG_4_Sampling_Frequency_Index = (byte)((bs[2] & 0x3C) >> 2);
//1: 1 channel: front-center
//2: 2 channels: front-left, front-right
//3: 3 channels: front-center, front-left, front-right
//4: 4 channels: front-center, front-left, front-right, back-center
//5: 5 channels: front-center, front-left, front-right, back-left, back-right
//6: 6 channels: front-center, front-left, front-right, back-left, back-right, LFE-channel
//7: 8 channels: front-center, front-left, front-right, side-left, side-right, back-left, back-right, LFE-channel
//通道数
MPEG_4_Channel_Configuration = (byte)(((bs[2] & 0x01) << 1) | ((bs[3] & 0xC0) >> 6));
//ADTS帧长度=ADTS头+AAC数据
Frame_Length = (ushort)(((bs[3] & 0x03) << 11) | ((bs[4] & 0xFF) << 3) | ((bs[5] & 0xE0) >> 5));
}
protected override void Parse(ref BitStreamReader bsr)
{
SpecMode = (SpectatorMode)bsr.ReadByte();
Target1 = bsr.ReadByte();
Target2 = bsr.ReadByte();
Unknown = bsr.ReadByte();
}
public DamageRemoteInfoMessage Deserialize(BlubSerializer serializer, BinaryReader reader)
{
var message = new DamageRemoteInfoMessage();
message.Target = reader.ReadUInt16();
message.AttackAttribute = reader.ReadEnum <AttackAttribute>();
message.GameTime = reader.ReadUInt32();
message.Source = reader.ReadUInt16();
message.Rotation = reader.ReadRotation();
message.Position = reader.ReadCompressedVector3();
message.Unk = reader.ReadCompressedFloat();
message.Damage = reader.ReadCompressedFloat();
var br = new BitStreamReader(reader.ReadBytes(2));
message.Flag1 = br.ReadByte(2);
message.Flag2 = br.ReadByte(1);
message.Flag3 = br.ReadByte(1);
message.Flag4 = br.ReadByte(1);
message.Flag5 = br.ReadByte(3);
message.Flag6 = br.ReadByte(4);
message.Flag7 = br.ReadByte(4);
return(message);
}
protected override void Parse(ref BitStreamReader bsr)
{
Client = bsr.ReadByte();
WantsToChat = bsr.ReadBool();
// TODO: fill out the rest of the UserMessage (don't know what SayText2
// is even but I guess it doesn't happen often at all since it hasn't broken a demo parse yet)
}
/// <summary>
/// Private helper used to read an int, short or byte (in reverse order) from the reader
/// and return an int
/// </summary>
/// <param name="reader"></param>
/// <param name="type"></param>
/// <returns></returns>
private int GetDataFromReader(BitStreamReader reader, GorillaEncodingType type)
{
switch (type)
{
case GorillaEncodingType.Int:
{
return((int)reader.ReadUInt32Reverse(Native.BitsPerInt));
}
case GorillaEncodingType.Short:
{
return((int)reader.ReadUInt16Reverse(Native.BitsPerShort));
}
case GorillaEncodingType.Byte:
{
return((int)reader.ReadByte(Native.BitsPerByte));
}
default:
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("bogus GorillaEncodingType passed to GetDataFromReader"));
}
}
}
protected override void Parse(ref BitStreamReader bsr)
{
MenuType = bsr.ReadUShort();
uint dataLen = bsr.ReadUInt();
_data = bsr.SplitAndSkip(dataLen);
}
/// <summary>
/// Encodes a single pixel row.
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public byte[] EncodeRow(byte[] data)
{
var reader = new BitStreamReader(new MemoryStream(data));
int counter = 0;
Initialize();
// iterate until stream ends
while (reader.Position < reader.Length)
{
if (reader.ReadBit() != m_State)
{
// write out counted pixels
WriteBits(counter);
// found state change
m_State = 1 - m_State;
// reset counter
counter = 0;
}
counter++;
}
// write the last data
if (counter > 0)
{
WriteBits(counter);
}
// return
return(Complete());
}
public object Deserialize(BinaryReader reader)
{
var message = new DamageInfoMessage();
message.Target = reader.ReadUInt16();
message.AttackAttribute = reader.ReadEnum <AttackAttribute>();
message.GameTime = reader.ReadUInt32();
message.Source = reader.ReadUInt16();
message.Unk5 = reader.ReadByte();
message.Rotation = reader.ReadRotation();
message.Position = reader.ReadCompressedVector3();
message.Unk6 = reader.ReadCompressedFloat();
message.Damage = reader.ReadCompressedFloat();
message.Unk8 = reader.ReadInt16();
message.Unk9 = reader.ReadUInt16();
var br = new BitStreamReader(reader.ReadBytes(3));
message.Flag1 = br.ReadByte(3);
message.Flag2 = br.ReadByte(2);
message.Flag3 = br.ReadByte(1);
message.Flag4 = br.ReadByte(1);
message.Flag5 = br.ReadByte(1);
message.Flag6 = br.ReadByte(1);
message.Flag7 = br.ReadByte(7);
message.IsCritical = br.ReadByte(4);
message.Flag9 = br.ReadByte(4);
return(message);
}
protected override void Parse(ref BitStreamReader bsr)
{
Command = (ShakeCommand)bsr.ReadByte();
Amplitude = bsr.ReadFloat();
Frequency = bsr.ReadFloat();
Duration = bsr.ReadFloat();
}
protected override void Parse(ref BitStreamReader bsr)
{
Portal = bsr.ReadEHandle();
Portalled = bsr.ReadEHandle();
bsr.ReadVector3(out NewPosition);
bsr.ReadVector3(out NewAngles);
}
static void TestBitStream()
{
BitStreamWriter writer = new BitStreamWriter(2);
BitStreamReader reader = new BitStreamReader();
// Resize, basic int
writer.WriteInt32(0x12345678);
writer.WriteInt8(39);
writer.WriteInt(2, 2);
writer.WriteInt(87, 7);
writer.WriteInt(33, 6);
reader.SetBytes(writer.DumpBytes());
Assert.IsTrue(reader.ReadInt32() == 0x12345678);
Assert.IsTrue(reader.ReadInt8() == 39);
Assert.IsTrue(reader.ReadInt(2) == 2);
Assert.IsTrue(reader.ReadInt(7) == 87);
Assert.IsTrue(reader.ReadInt(6) == 33);
// Concatnat bits
// 1, 0000010 11000011, 1
// should be 00000101 10000110 00000011
writer.WriteBool(true);
writer.WriteInt16(707);
writer.WriteBool(true);
var data = writer.DumpBytes();
Assert.IsTrue(data.Length == 3);
Assert.IsTrue(data[0] == 0x05);
Assert.IsTrue(data[1] == 0x86);
Assert.IsTrue(data[2] == 0x03);
reader.SetBytes(data);
Assert.IsTrue(reader.ReadBool());
Assert.IsTrue(reader.ReadInt16() == 707);
Assert.IsTrue(reader.ReadBool());
// float
writer.WriteBool(false);
writer.WriteFloat(12345.012345f);
writer.WriteBool(true);
reader.SetBytes(writer.DumpBytes());
Assert.IsFalse(reader.ReadBool());
Assert.IsTrue(reader.ReadFloat() == 12345.012345f);
Assert.IsTrue(reader.ReadBool());
// vector3, quaternion
Vector3 vec = new Vector3(-0.51231f, 0.113123f, 1.1231123f);
Quaternion q = new Quaternion(-0.123f, 0.345f, 0.678f, -0.23f);
q.Normalize();
writer.WriteQuaternionRot(q);
writer.WriteBool(false);
writer.WriteVector3(Vector3.one);
writer.WriteVector3(vec);
reader.SetBytes(writer.DumpBytes());
Assert.IsTrue(IsApprox(reader.ReadQuaternionRot(), q));
Assert.IsFalse(reader.ReadBool());
Assert.IsTrue(reader.ReadVector3() == Vector3.one);
Assert.IsTrue(reader.ReadVector3() == vec);
}
// I don't think I've seen this in demos yet, I'll just log it for now and deal with it later
protected override void Parse(ref BitStreamReader bsr)
{
NeedsDecoder = bsr.ReadBool();
ushort len = bsr.ReadUShort();
_props = bsr.Split(len);
DemoRef.LogError($"unprocessed {GetType().Name} message"); // todo se2007/engine/dt_send_eng.cpp line 800
}
protected override void Parse(ref BitStreamReader bsr)
{
Angle = new Vector3 {
X = bsr.ReadBitAngle(16),
Y = bsr.ReadBitAngle(16),
Z = bsr.ReadBitAngle(16),
};
}
public void SetLength_NotSupported()
{
var data = new byte[] { 0x3c, 0x4b, 0x1a, 0x54, 0x22, 0x10, 0xaf, 0x89, 0x97, 0x65 };
var stream = new MemoryStream(data);
var reader = new BitStreamReader(stream);
reader.SetLength(12);
}
public static Vector2Int ReadVector2Int(this BitStreamReader reader)
{
return(new Vector2Int
{
x = reader.ReadInt32(),
y = reader.ReadInt32()
});
}
public void WriteBuffer_NotSupported()
{
var data = new byte[] { 0x3c, 0x4b, 0x1a, 0x54, 0x22, 0x10, 0xaf, 0x89, 0x97, 0x65 };
var stream = new MemoryStream(data);
var reader = new BitStreamReader(stream);
reader.Write(new byte[] { 0x35, 0x12 }, 0, 2);
}
public static Vector2 ReadVector2(this BitStreamReader reader)
{
return(new Vector2
{
x = reader.ReadFloat32(),
y = reader.ReadFloat32()
});
}
public MeasurementStreamFileReading()
{
Value = new UnionValues();
m_points = new List<PointMetaData>();
m_buffer = new ByteBuffer(4096);
m_bitStream = new BitStreamReader(this);
NewMeasurementRegisteredMetadata = new byte[16];
UserCommandData = new byte[16];
}
public static byte[] Decompress(BinaryReader input, int expectedSize)
{
Contract.Requires(input != null);
Contract.Requires(expectedSize >= 0);
Contract.Ensures(Contract.Result<byte[]>() != null);
var bitStream = new BitStreamReader(input);
var compressionType = (PkLibCompressionType)input.ReadByte();
if (compressionType != PkLibCompressionType.Binary && compressionType != PkLibCompressionType.ASCII)
throw new InvalidDataException("Invalid compression type: {0}".Interpolate(compressionType));
var dictSizeBits = input.ReadByte();
if (dictSizeBits < 4 || dictSizeBits > 6)
throw new InvalidDataException("Invalid dictionary size: {0}".Interpolate(dictSizeBits));
var outputBuffer = new byte[expectedSize];
using (var outputStream = new MemoryStream(outputBuffer))
{
int instruction;
while ((instruction = DecodeLiteral(bitStream, compressionType)) != -1)
{
if (instruction >= 0x100)
{
// If instruction is greater than 0x100, it means "repeat n - 0xfe bytes".
var copyLength = instruction - 0xfe;
var moveBack = DecodeDistance(bitStream, copyLength, dictSizeBits);
if (moveBack == 0)
break;
var source = (int)outputStream.Position - moveBack;
while (copyLength-- > 0)
outputStream.WriteByte(outputBuffer[source++]);
}
else
outputStream.WriteByte((byte)instruction);
}
if (outputStream.Position == expectedSize)
return outputBuffer;
return outputStream.ToArray();
}
}
/// <summary>
/// Decode
/// </summary>
/// <param name="data"></param>
/// <param name="extra"></param>
/// <param name="reader"></param>
/// <returns>number of bits decoded, 0 for error</returns>
internal void Decode(ref int data, ref int extra, BitStreamReader reader)
{
// Find the prefix length
byte prefIndex = 0;
while (reader.ReadBit())
{
prefIndex++;
}
// First indicate there is no extra data
extra = 0;
// More efficient for 0
if (0 == prefIndex)
{
data = 0;
return;
}
else if (prefIndex < _huffBits.GetSize())
{
// Find the data lenght
uint nDataLen = _huffBits.GetBitsAtIndex(prefIndex);
// Extract the offset data by lower dound with sign bit at LSB
long nData = reader.ReadUInt64((int)(byte)nDataLen);
// Find the sign bit
bool bNeg = ((nData & 0x01) != 0);
// Construct the data
nData = (nData >> 1) + _mins[prefIndex];
// Adjust the sign bit
data = bNeg ? -((int)nData) : (int)nData;
// return the bit count read from stream
return;
}
else if (prefIndex == _huffBits.GetSize())
{
// This is the special prefix for extra data.
// Decode the prefix first
int extra2 = 0;
int extra2Ignored = 0;
Decode(ref extra2, ref extra2Ignored, reader);
extra = extra2;
// Following is the actual data
int data2 = 0;
Decode(ref data2, ref extra2Ignored, reader);
data = data2;
return;
}
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("invalid huffman encoded data"));
}
/// <summary>
/// Uncompress - uncompress a byte[] into an int[] of point data (x,x,x,x,x)
/// </summary>
/// <param name="bitCount">The number of bits each element uses in input</param>
/// <param name="input">compressed data</param>
/// <param name="inputIndex">index to begin decoding at</param>
/// <param name="dtxf">data xf, can be null</param>
/// <param name="outputBuffer">output buffer that is prealloc'd to write to</param>
/// <param name="outputBufferIndex">the index of the output buffer to write to</param>
internal uint Uncompress(int bitCount, byte[] input, int inputIndex, DeltaDelta dtxf, int[] outputBuffer, int outputBufferIndex)
{
if (null == input)
{
throw new ArgumentNullException("input");
}
if (inputIndex >= input.Length)
{
throw new ArgumentOutOfRangeException("inputIndex");
}
if (null == outputBuffer)
{
throw new ArgumentNullException("outputBuffer");
}
if (outputBufferIndex >= outputBuffer.Length)
{
throw new ArgumentOutOfRangeException("outputBufferIndex");
}
if (bitCount < 0)
{
throw new ArgumentOutOfRangeException("bitCount");
}
// Adjust bit count if 0 passed in
if (bitCount == 0)
{
//adjust if the bitcount is 0
//(this makes bitCount 32)
bitCount = (int)(Native.SizeOfInt << 3);
}
// Test whether the items are signed. For unsigned number, we don't need mask
// If we are trying to compress signed long values with bit count = 5
// The mask will be 1111 1111 1111 0000. The way it is done is, if the 5th
// bit is 1, the number is negative numbe, othrwise it's positive. Testing
// will be non-zero, ONLY if the 5th bit is 1, in which case we OR with the mask
// otherwise we leave the number as it is.
uint bitMask = (unchecked((uint)~0) << (bitCount - 1));
uint bitData = 0;
BitStreamReader reader = new BitStreamReader(input, inputIndex);
if(dtxf != null)
{
while (!reader.EndOfStream)
{
bitData = reader.ReadUInt32(bitCount);
// Construct the item
bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData;
int result = dtxf.InverseTransform((int)bitData, 0);
Debug.Assert(outputBufferIndex < outputBuffer.Length);
outputBuffer[outputBufferIndex++] = result;
if (outputBufferIndex == outputBuffer.Length)
{
//only write as much as the outputbuffer can hold
//this is assumed by calling code
break;
}
}
}
else
{
while (!reader.EndOfStream)
{
bitData = reader.ReadUInt32(bitCount);
// Construct the item
bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData;
Debug.Assert(outputBufferIndex < outputBuffer.Length);
outputBuffer[outputBufferIndex++] = (int)bitData;
if (outputBufferIndex == outputBuffer.Length)
{
//only write as much as the outputbuffer can hold
//this is assumed by calling code
break;
}
}
}
// Calculate how many bytes were read from input buffer
return (uint)((outputBuffer.Length * bitCount + 7) >> 3);
}
/// <summary>
/// Uncompress
/// </summary>
/// <param name="dtxf"></param>
/// <param name="input"></param>
/// <param name="startIndex"></param>
/// <param name="outputBuffer"></param>
internal uint Uncompress(DataXform dtxf, byte[] input, int startIndex, int[] outputBuffer)
{
Debug.Assert(input != null);
Debug.Assert(input.Length >= 2);
Debug.Assert(startIndex == 1);
Debug.Assert(outputBuffer != null);
Debug.Assert(outputBuffer.Length != 0);
BitStreamReader reader = new BitStreamReader(input, startIndex);
int xfExtra = 0, xfData = 0;
int outputBufferIndex = 0;
if (null != dtxf)
{
dtxf.ResetState();
while (!reader.EndOfStream)
{
Decode(ref xfData, ref xfExtra, reader);
int uncompressed = dtxf.InverseTransform(xfData, xfExtra);
Debug.Assert(outputBufferIndex < outputBuffer.Length);
outputBuffer[outputBufferIndex++] = uncompressed;
if (outputBufferIndex == outputBuffer.Length)
{
//only write as much as the outputbuffer can hold
//this is assumed by calling code
break;
}
}
}
else
{
while (!reader.EndOfStream)
{
Decode(ref xfData, ref xfExtra, reader);
Debug.Assert(outputBufferIndex < outputBuffer.Length);
outputBuffer[outputBufferIndex++] = xfData;
if (outputBufferIndex == outputBuffer.Length)
{
//only write as much as the outputbuffer can hold
//this is assumed by calling code
break;
}
}
}
return (uint)((reader.CurrentIndex + 1) - startIndex); //we include startIndex in the read count
}
/// <summary>
/// Compresses property data which is already in the form of a byte[]
/// into a compressed byte[]
/// </summary>
/// <param name="input">byte[] data ready to be compressed</param>
/// <param name="compression">the compression to use</param>
/// <returns></returns>
internal byte[] CompressPropertyData(byte[] input, byte compression)
{
List<byte> compressedData = new List<byte>(input.Length + 1); //reasonable default based on profiling.
//leave room at the beginning of
//compressedData for the compression header byte
compressedData.Add((byte)0);
if (DefaultCompression == (DefaultCompression & compression))
{
compression = this.GorillaCodec.FindPropAlgoByte(input);
}
//validate that we never lzencode
if (LempelZiv == (compression & LempelZiv))
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("Invalid compression specified or computed by FindPropAlgoByte"));
}
//determine what the optimal way to compress the data is. Should we treat
//the byte[] as a series of Int's, Short's or Byte's?
int countPerItem = 0, bitCount = 0, padCount = 0;
this.GorillaCodec.GetPropertyBitCount(compression, ref countPerItem, ref bitCount, ref padCount);
Debug.Assert(countPerItem == 4 || countPerItem == 2 || countPerItem == 1);
GorillaEncodingType type = GorillaEncodingType.Byte;
int unitCount = input.Length;
if (countPerItem == 4)
{
type = GorillaEncodingType.Int;
unitCount >>= 2;
}
else if (countPerItem == 2)
{
type = GorillaEncodingType.Short;
unitCount >>= 1;
}
BitStreamReader reader = new BitStreamReader(input);
//encode, gorilla style
this.GorillaCodec.Compress(bitCount, //the max count of bits required for each int
reader, //the reader, which can read int, byte, short
type, //informs how the reader reads
unitCount, //just how many items do we need to compress?
compressedData); //a ref to the compressed data that will be written to
compressedData[0] = compression;
return compressedData.ToArray();
}
private static int DecodeLiteral(BitStreamReader bitStream, PkLibCompressionType compressionType)
{
Contract.Requires(bitStream != null);
// Return values:
// 0x000 to 0x0ff: One byte from compressed file.
// 0x100 to 0x305: Copy previous block (0x100 = 1 byte).
// -1: End of stream.
switch (bitStream.ReadBits(1))
{
case -1:
return -1;
case 1:
// The next bits are positions in buffers.
int pos = _sPosition2[bitStream.PeekByte()];
// Skip the bits we just used.
var numBits = _sLenBits[pos];
Contract.Assume(numBits < BitStreamReader.MaxBitCount);
if (bitStream.ReadBits(numBits) == -1)
return -1;
var nBits = _sExLenBits[pos];
if (nBits != 0)
{
Contract.Assume(nBits < BitStreamReader.MaxBitCount);
var val2 = bitStream.ReadBits(nBits);
if (val2 == -1 && (pos + val2 != 0x10e))
return -1;
pos = _sLenBase[pos] + val2;
}
return pos + 0x100; // Return number of bytes to repeat.
case 0:
if (compressionType == PkLibCompressionType.Binary)
return bitStream.ReadBits(sizeof(byte) * 8);
// TODO: Implement ASCII mode.
throw new NotImplementedException("ASCII mode is not yet implemented.");
default:
return 0;
}
}
/// <summary>
/// Decompresses property data (from a compressed byte[] to an uncompressed byte[])
/// </summary>
/// <param name="input">The byte[] to decompress</param>
/// <returns></returns>
internal byte[] DecompressPropertyData(byte[] input)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (input.Length < 2)
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("input.Length must be at least 2"));
}
byte compression = input[0];
int inputIndex = 1;
if (LempelZiv == (compression & LempelZiv))
{
if (0 != (compression & (~LempelZiv)))
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("bogus isf, we don't decompress property data with lz"));
}
return this.LZCodec.Uncompress(input, inputIndex);
}
else
{
//gorilla
//determine what the way to uncompress the data. Should we treat
//the byte[] as a series of Int's, Short's or Byte's?
int countPerItem = 0, bitCount = 0, padCount = 0;
this.GorillaCodec.GetPropertyBitCount(compression, ref countPerItem, ref bitCount, ref padCount);
Debug.Assert(countPerItem == 4 || countPerItem == 2 || countPerItem == 1);
GorillaEncodingType type = GorillaEncodingType.Byte;
if (countPerItem == 4)
{
type = GorillaEncodingType.Int;
}
else if (countPerItem == 2)
{
type = GorillaEncodingType.Short;
}
//determine how many units (of int, short or byte) that there are to decompress
int unitsToDecode = ((input.Length - inputIndex << 3) / bitCount) - padCount;
BitStreamReader reader = new BitStreamReader(input, inputIndex);
return this.GorillaCodec.Uncompress(bitCount, reader, type, unitsToDecode);
}
}
/// <summary>
/// FindPropAlgoByte - find the best way to compress the input array
/// </summary>
/// <param name="input"></param>
internal byte FindPropAlgoByte(byte[] input)
{
// Empty buffer case
if(0 == input.Length)
{
return 0;
}
// We test for int's only if the data size is multiple of 4
int countOfInts = ((0 == (input.Length & 0x03)) ? input.Length >> 2 : 0);
BitStreamReader intReader = null;
if (countOfInts > 0)
{
intReader = new BitStreamReader(input);
}
// We test for shorts's if datasize is multiple of 2
int countOfShorts = ((0 == (input.Length & 0x01)) ? input.Length >> 1 : 0);
BitStreamReader shortReader = null;
if (countOfShorts > 0)
{
shortReader = new BitStreamReader(input);
}
// Min Max variables for different data type
int maxInt = 0, minInt = 0;
// Unsigned min vals
ushort maxShort = 0;
// byte min/max
byte maxByte = input[0];
// Find min/max of all data
// This loop covers:
// All of int data, if there is any
// First half of Word data, if there is int data, there MUST be word data
// First quarter of byte data.
uint n = 0;
for(n = 0; n < countOfInts; ++n)
{
Debug.Assert(intReader != null);
Debug.Assert(shortReader != null);
maxByte = Math.Max(input[n], maxByte);
maxShort = Math.Max((ushort)shortReader.ReadUInt16Reverse(Native.BitsPerShort), maxShort);
UpdateMinMax((int)intReader.ReadUInt32Reverse(Native.BitsPerInt), ref maxInt, ref minInt);
}
// This loop covers:
// Second half of short data, if there were int data,
// or all of short data, if there were no int data
// Upto half of byte data
for(; n < countOfShorts; ++n)
{
Debug.Assert(shortReader != null);
maxByte = Math.Max(input[n], maxByte);
maxShort = Math.Max((ushort)shortReader.ReadUInt16Reverse(Native.BitsPerShort), maxShort);
}
// This loop covers last half of byte data if word data existed
// or, all of bytes data
for (; n < input.Length; ++n)
{
maxByte = Math.Max(input[n], maxByte);
}
// Which one gives the best result?
int bitCount = 1;
// Find the Math.Abs max for byte
uint ulAbsMax = (uint)maxByte;
// Find the number of bits required to encode that number
while ((0 != (ulAbsMax >> bitCount)) && (bitCount < (uint)Native.BitsPerByte))
{
bitCount++;
}
// Also compute the padding required
int padCount = ((((~(bitCount * input.Length)) & 0x07) + 1) & 0x07) / bitCount;
// Compare the result with word partition
if (countOfShorts > 0)
{
int shortBitCount = 1;
// Find the Math.Abs max for word
ulAbsMax = (uint)maxShort;
// Find the number of bits required to encode that number
while ((0 != (ulAbsMax >> shortBitCount)) && (shortBitCount < (uint)Native.BitsPerShort))
{
shortBitCount++;
}
// Determine which scheme requires lesser number of bytes
if (shortBitCount < (bitCount << 1))
{
bitCount = shortBitCount;
padCount = ((((~(bitCount * countOfShorts)) & 0x07) + 1) & 0x07) / bitCount;
}
else
{
countOfShorts = 0;
}
}
// Compare the best with int
if(countOfInts > 0)
{
int intBitCount = 0;
// Find the Math.Abs max for int
ulAbsMax = (uint)Math.Max(MathHelper.AbsNoThrow(minInt), MathHelper.AbsNoThrow(maxInt));
// Find the number of bits required to encode that number
while ((0 != (ulAbsMax >> intBitCount)) && (31 > intBitCount))
{
intBitCount++;
}
// Adjust for the sign bit
intBitCount++;
// Determine which one is better
if (intBitCount < ((0 < countOfShorts) ? (bitCount << 1) : (bitCount << 2)))
{
bitCount = intBitCount;
padCount = ((((~(bitCount * countOfInts)) & 0x07) + 1) & 0x07) / bitCount;
// Set the countOfShorts to 0 to indicate int wins over word
countOfShorts = 0;
}
else
{
countOfInts = 0;
}
}
// AlgoByte starts with 000, 001 and 01 for byte, word and int correspondingly
byte algorithmByte = (byte)((0 < countOfInts) ? 0x40 : ((0 < countOfShorts) ? 0x20 : 0x00));
// If byte, and bitCount is 8, we revert use 0 as algo byte
if ((8 == bitCount) && (0 == (countOfInts + countOfShorts)))
{
algorithmByte = 0;
}
// If bitCount is more than 7, we add 16 to make the index
else if (bitCount > 7)
{
algorithmByte |= (byte)(16 + bitCount);
}
// Otherwise, we find the index from the table
else
{
algorithmByte |= (byte)(_gorIndexOffset[bitCount] + padCount);
}
return algorithmByte;
}
/// <summary>
/// Uncompress - uncompress the byte[] in the reader to a byte[] to return
/// </summary>
/// <param name="bitCount">number of bits each element is compressed to</param>
/// <param name="reader">a reader over the compressed byte[]</param>
/// <param name="encodingType">int, short or byte?</param>
/// <param name="unitsToDecode">number of logical units to decode</param>
/// <returns>Uncompressed byte[]</returns>
internal byte[] Uncompress(int bitCount, BitStreamReader reader, GorillaEncodingType encodingType, int unitsToDecode)
{
if (null == reader)
{
throw new ArgumentNullException("reader");
}
if (bitCount < 0)
{
throw new ArgumentOutOfRangeException("bitCount");
}
if (unitsToDecode < 0)
{
throw new ArgumentOutOfRangeException("unitsToDecode");
}
int bitsToWrite = 0;
// Test whether the items are signed. For unsigned number, we don't need mask
// If we are trying to compress signed long values with bit count = 5
// The mask will be 1111 1111 1111 0000. The way it is done is, if the 5th
// bit is 1, the number is negative numbe, othrwise it's positive. Testing
// will be non-zero, ONLY if the 5th bit is 1, in which case we OR with the mask
// otherwise we leave the number as it is.
uint bitMask = 0;
//adjust if the bitcount is 0
//(this makes bitCount 32)
switch (encodingType)
{
case GorillaEncodingType.Int:
{
if (bitCount == 0)
{
bitCount = Native.BitsPerInt;
}
bitsToWrite = Native.BitsPerInt;
//we decode int's as unsigned, so we need to create a mask
bitMask = (unchecked((uint)~0) << (bitCount - 1));
break;
}
case GorillaEncodingType.Short:
{
if (bitCount == 0)
{
bitCount = Native.BitsPerShort;
}
bitsToWrite = Native.BitsPerShort;
//shorts are decoded as unsigned values, no mask required
bitMask = 0;
break;
}
case GorillaEncodingType.Byte:
{
if (bitCount == 0)
{
bitCount = Native.BitsPerByte;
}
bitsToWrite = Native.BitsPerByte;
//bytes are decoded as unsigned values, no mask required
bitMask = 0;
break;
}
default:
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("bogus GorillaEncodingType passed to Uncompress"));
}
}
List<byte> output = new List<byte>((bitsToWrite / 8) * unitsToDecode);
BitStreamWriter writer = new BitStreamWriter(output);
uint bitData = 0;
while (!reader.EndOfStream && unitsToDecode > 0)
{
//we're going to cast to an uint anyway, just read as one
bitData = reader.ReadUInt32(bitCount);
// Construct the item
bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData;
writer.WriteReverse(bitData, bitsToWrite);
unitsToDecode--;
}
return output.ToArray();
}
private static int DecodeDistance(BitStreamReader bitStream, int length, int dictSizeBits)
{
Contract.Requires(bitStream != null);
Contract.Requires(length >= 0);
Contract.Requires(dictSizeBits >= 0);
Contract.Requires(dictSizeBits < BitStreamReader.MaxBitCount);
if (bitStream.EnsureBits(8) == false)
return 0;
var pos = (int)_sPosition1[bitStream.PeekByte()];
var skip = _sDistBits[pos]; // Number of bits to skip.
// Skip the appropriate number of bits
Contract.Assume(skip < BitStreamReader.MaxBitCount);
if (bitStream.ReadBits(skip) == -1)
return 0;
if (length == 2)
{
if (bitStream.EnsureBits(2) == false)
return 0;
pos = (pos << 2) | bitStream.ReadBits(2);
}
else
{
if (bitStream.EnsureBits(dictSizeBits) == false)
return 0;
pos = ((pos << dictSizeBits)) | bitStream.ReadBits(dictSizeBits);
}
return pos + 1;
}
/// <summary>
/// Private helper used to read an int, short or byte (in reverse order) from the reader
/// and return an int
/// </summary>
/// <param name="reader"></param>
/// <param name="type"></param>
/// <returns></returns>
private int GetDataFromReader(BitStreamReader reader, GorillaEncodingType type)
{
switch (type)
{
case GorillaEncodingType.Int:
{
return (int)reader.ReadUInt32Reverse(Native.BitsPerInt);
}
case GorillaEncodingType.Short:
{
return (int)reader.ReadUInt16Reverse(Native.BitsPerShort);
}
case GorillaEncodingType.Byte:
{
return (int)reader.ReadByte(Native.BitsPerByte);
}
default:
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("bogus GorillaEncodingType passed to GetDataFromReader"));
}
}
}
/// <summary>
/// Compress - compresses the byte[] being read by the BitStreamReader into compressed data
/// </summary>
/// <param name="bitCount">the number of bits to use for each element</param>
/// <param name="reader">a reader over the byte[] to compress</param>
/// <param name="encodingType">int, short or byte?</param>
/// <param name="unitsToEncode">number of logical units to encoded</param>
/// <param name="compressedData">output write buffer</param>
internal void Compress(int bitCount, BitStreamReader reader, GorillaEncodingType encodingType, int unitsToEncode, List<byte> compressedData)
{
if (null == reader || null == compressedData)
{
throw new ArgumentNullException(StrokeCollectionSerializer.ISFDebugMessage("reader or compressedData was null in compress"));
}
if (bitCount < 0)
{
throw new ArgumentOutOfRangeException("bitCount");
}
if (unitsToEncode < 0)
{
throw new ArgumentOutOfRangeException("unitsToEncode");
}
if (bitCount == 0)
{
//adjust if the bitcount is 0
//(this makes bitCount 32)
switch (encodingType)
{
case GorillaEncodingType.Int:
{
bitCount = Native.BitsPerInt;
break;
}
case GorillaEncodingType.Short:
{
bitCount = Native.BitsPerShort;
break;
}
case GorillaEncodingType.Byte:
{
bitCount = Native.BitsPerByte;
break;
}
default:
{
throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("bogus GorillaEncodingType passed to compress"));
}
}
}
//have the writer adapt to the List<byte> passed in and write to it
BitStreamWriter writer = new BitStreamWriter(compressedData);
while (!reader.EndOfStream && unitsToEncode > 0)
{
int data = GetDataFromReader(reader, encodingType);
writer.Write((uint)data, bitCount);
unitsToEncode--;
}
}
