public static byte[] Decode(byte[] encodedData, Dictionary <BitsWithLength, byte> decodeTable, long bitsCount)
{
var result = new List <byte>();
var sample = new BitsWithLength {
Bits = 0, BitsCount = 0
};
for (var byteNum = 0; byteNum < encodedData.Length; byteNum++)
{
var b = encodedData[byteNum];
for (var bitNum = 0; bitNum < 8 && byteNum * 8 + bitNum < bitsCount; bitNum++)
{
sample.Bits = (sample.Bits << 1) + ((b & (1 << (8 - bitNum - 1))) != 0 ? 1 : 0);
sample.BitsCount++;
if (decodeTable.TryGetValue(sample, out var decodedByte))
{
result.Add(decodedByte);
sample.BitsCount = 0;
sample.Bits = 0;
}
}
}
return(result.ToArray());
}
private static IEnumerable <byte> DecodeChunk(byte[] data, Dictionary <BitsWithLength, byte> decodeTable, int totalBitsCount)
{
var result = new List <byte>();
var bits = (int)0;
var bitsCount = (int)0;
for (var byteNum = 0; byteNum < data.Length; byteNum++)
{
var b = data[byteNum];
for (var bitNum = 0; bitNum < 8 && byteNum * 8 + bitNum < totalBitsCount; bitNum++)
{
bits = (int)((bits << 1) + ((b & (1 << (8 - bitNum - 1))) != 0 ? 1 : 0));
bitsCount++;
var key = new BitsWithLength(bits, bitsCount);
byte decodedByte;
if (decodeTable.TryGetValue(key, out decodedByte))
{
result.Add(decodedByte);
bitsCount = 0;
bits = 0;
}
}
}
return(result);
}
public static byte[] Encode(byte[] data, out Dictionary <BitsWithLength, byte> decodeTable, Options options)
{
var frequences = ParallelCalcFrequences(data);
var root = BuildHuffmanTree(frequences);
var encodeTable = new BitsWithLength[byte.MaxValue + 1];
FillEncodeTable(root, encodeTable);
var chunkSize = 32 * 1024;//TODO optimize decoding with array preallocation
var dataSize = data.Length;
var chunkCount = dataSize / chunkSize + (dataSize % chunkSize == 0 ? 0 : 1);
var chunks = Enumerable.Range(0, chunkCount)
.Select(
i =>
new
{
Data = data.Skip(i * chunkSize).Take(chunkSize),
BitsBuffer = new BitsBuffer()
})
.ToList();
Parallel.ForEach(chunks, new ParallelOptions {
MaxDegreeOfParallelism = options.MaxDegreeOfParallelism
}, p =>
{
foreach (var b in p.Data)
{
p.BitsBuffer.Add(encodeTable[b]);
}
});
decodeTable = CreateDecodeTable(encodeTable);
return(chunks.SelectMany(chunk => chunk.BitsBuffer.ToArray()).ToArray());
}
public static byte[] Encode(IEnumerable <byte> data, out Dictionary <BitsWithLength, byte> decodeTable, out long bitsCount)
{
var frequences = CalcFrequences(data, byte.MaxValue + 1);
var root = BuildHuffmanTree(frequences);
var encodeTable = new BitsWithLength[frequences.Length];
FillEncodeTable(root, encodeTable);
var bitsBuffer = new BitsBuffer();
foreach (var b in data)
{
bitsBuffer.Add(encodeTable[b]);
}
decodeTable = CreateDecodeTable(encodeTable);
return(bitsBuffer.ToArray(out bitsCount));
}
public void Add(BitsWithLength bitsWithLength)
{
var bitsCount = bitsWithLength.BitsCount;
var bits = bitsWithLength.Bits;
int neededBits = 8 - unfinishedBits.BitsCount;
while (bitsCount >= neededBits)
{
bitsCount -= neededBits;
buffer.Add((byte)((unfinishedBits.Bits << neededBits) + (bits >> bitsCount)));
bits = bits & ((1 << bitsCount) - 1);
unfinishedBits.Bits = 0;
unfinishedBits.BitsCount = 0;
neededBits = 8;
}
unfinishedBits.BitsCount += bitsCount;
unfinishedBits.Bits = (unfinishedBits.Bits << bitsCount) + bits;
}
public void Add(BitsWithLength bitsWithLength)
{
var bitsCount = bitsWithLength.BitsCount;
var bits = bitsWithLength.Bits;
var neededBits = 8 - this.unfinishedBits.BitsCount;
while (bitsCount >= neededBits)
{
bitsCount -= neededBits;
this.buffer.Add((byte)((this.unfinishedBits.Bits << neededBits) + (bits >> bitsCount)));
bits = bits & ((1 << bitsCount) - 1);
this.unfinishedBits.Bits = 0;
this.unfinishedBits.BitsCount = 0;
neededBits = 8;
}
this.unfinishedBits.BitsCount += bitsCount;
this.unfinishedBits.Bits = (this.unfinishedBits.Bits << bitsCount) + bits;
}
public bool Equals(BitsWithLength other)
{
return(Bits == other.Bits && BitsCount == other.BitsCount);
}
