public void CompressContentTest()
{
Stream s = File.OpenWrite(@"D:\MFF\ZS_2019\c#_repos\Huffman1\Huffman1Tests\bin\Debug\netcoreapp3.0\TestFiles\Huff2\Outs\Compressed.out");
BinaryWriter writer = new BinaryWriter(s);
Stream ins = File.OpenRead(@"D:\MFF\ZS_2019\c#_repos\Huffman1\Huffman1Tests\bin\Debug\netcoreapp3.0\TestFiles\Huff2\Ins\simple4.in");
BinaryReader reader = new BinaryReader(ins);
HuffmanReader HuffReader = new HuffmanReader(reader);
bool NotEmpty = HuffReader.ReadFileUsingBuffer();
//Build Huffman tree
TreeBuilder Builder = new TreeBuilder(HuffReader.ProvideSymbolsWihtWeights());
Node Root = Builder.BuildHuffTree();
ins.Seek(0, 0);
HuffCompresor.CompressContent(Root, writer, reader);
writer.Close();
reader.Close();
bool same = Utils.FileDiff(@"D:\MFF\ZS_2019\c#_repos\Huffman1\Huffman1Tests\bin\Debug\netcoreapp3.0\TestFiles\Huff2\Outs\Compressed.huff", @"D:\MFF\ZS_2019\c#_repos\Huffman1\Huffman1Tests\bin\Debug\netcoreapp3.0\TestFiles\Huff2\Outs\Compressed.out");
Assert.IsTrue(same);
}
private static byte[] getEntry(long id, long key)
{
string datasetQ = "select * from dataset where datasetId=" + id + " and datasetKey=" + key;
using (SqliteCommand datasetQcmd = new SqliteCommand(datasetQ, origc))
{
using (SqliteDataReader reader = datasetQcmd.ExecuteReader(System.Data.CommandBehavior.Default))
{
while (reader.Read())
{
byte[] compressedData = (byte[])reader.GetValue(5);
// cache the huffman readers to save having to compute the huffman trees every time and also we don't have to read the frequency tables every time
HuffmanReader huffreader;
if (!dsHuffmanreaders.TryGetValue(id, out huffreader))
{
huffreader = new HuffmanReader(getFreqData(origc, id));
dsHuffmanreaders[id] = huffreader;
}
byte[] data = getData(compressedData, huffreader);
return(data);
}
}
}
Debug.LogError("Failed to get result for " + id + ":" + key);
return(new byte[0]);
}
public override void Decode(Stream input, Stream output)
{
using (var br = new BitReader(input, BitOrder.MostSignificantBitFirst, 4, ByteOrder.LittleEndian))
{
var uncompressedSize = br.ReadInt32() >> 8;
br.ReadByte();
HuffmanReader.BuildTree(br);
SetupDisplacementTable(br, 4);
while (output.Length < uncompressedSize)
{
if (br.ReadBit() == 1)
{
var displacement = GetDisplacement(br, br.ReadBits <byte>(2));
var matchLength = br.ReadBits <byte>(4) + 3;
ReadDisplacement(output, displacement, matchLength, 1);
}
else
{
ReadHuffmanValues(br, output, 1, 1);
}
}
}
}
public void CodeBytesTest() //no assert!
{
Stream ins = File.OpenRead(@"D:\MFF\ZS_2019\c#_repos\Huffman1\Huffman1Tests\bin\Debug\netcoreapp3.0\TestFiles\Huff2\Ins\simple4.in");
BinaryReader reader = new BinaryReader(ins);
HuffmanReader HuffReader = new HuffmanReader(reader);
bool NotEmpty = HuffReader.ReadFileUsingBuffer();
//Build Huffman tree
TreeBuilder Builder = new TreeBuilder(HuffReader.ProvideSymbolsWihtWeights());
Node Root = Builder.BuildHuffTree();
TreePrinter.PrintNiceTree(Root, 0);
Stack <byte> path = new Stack <byte>();
HuffCompresor.CodeBytes(Root, path);
for (int i = 0; i < HuffCompresor.Codes.Length; i++)
{
if (HuffCompresor.Codes[i] != null)
{
Console.Write($"{i}: ");
for (int j = 0; j < HuffCompresor.Codes[i].Length; j++)
{
Console.Write($"{HuffCompresor.Codes[i][j]} ");
}
Console.WriteLine();
}
}
}
public void EmptyInputTest()
{
Stream s = File.OpenRead(@"TestFiles\Ins\EmptyInput.bin");
BinaryReader br = new BinaryReader(s);
HuffmanReader reader = new HuffmanReader(br);
bool b = reader.ReadFile();
Assert.IsFalse(b);
}
public override void Decode(Stream input, Stream output)
{
using (var br = new BitReader(input, BitOrder.MsbFirst, 4, ByteOrder.LittleEndian))
{
var uncompressedSize = br.ReadInt32() >> 8;
br.ReadByte();
HuffmanReader.BuildTree(br);
while (output.Length < uncompressedSize)
{
ReadHuffmanValues(br, output, 1, 1);
}
}
}
public override void Decode(Stream input, Stream output)
{
using (var br = new BitReader(input, BitOrder.MsbFirst, 4, ByteOrder.LittleEndian))
{
var uncompressedSize = br.ReadInt32() >> 8;
br.ReadByte();
HuffmanReader.BuildTree(br);
SetupDisplacementTable(br, 2);
while (output.Length < uncompressedSize)
{
int matchLength;
int displacement;
var value0 = br.ReadBits <byte>(2);
if (value0 < 2) // cmp value0, #2 -> bcc
{
// CC4
displacement = GetDisplacement(br, value0);
matchLength = br.ReadBits <byte>(6) + 3;
// Goto EC4
}
else
{
if (value0 == 2)
{
// CDC
matchLength = br.ReadBits <byte>(6) + 1;
ReadHuffmanValues(br, output, matchLength, 1);
continue;
}
// value0 == 3
// CA4
matchLength = br.ReadBits <byte>(6) + 1;
displacement = 1;
output.WriteByte(br.ReadBits <byte>(8)); // read static 8 bit; we don't read a huffman value here
// Goto EC4
}
// EC4
ReadDisplacement(output, displacement, matchLength, 1);
}
}
}
private static byte[] getData(byte[] compressedData, HuffmanReader reader)
{
MemoryStream mdata = new MemoryStream(compressedData);
int uncompressedSize = RiftAssets.Util.readUnsignedLeb128_X(mdata);
byte[] dataOutput = new byte[uncompressedSize];
MemoryStream compressedD = new MemoryStream();
CopyStream(mdata, compressedD);
compressedD.Seek(0, SeekOrigin.Begin);
byte[] newCompressed = compressedD.ToArray();
return(reader.read(newCompressed, newCompressed.Length, uncompressedSize));
}
public void process(byte[] cdsData, Action <String> progress)
{
logger.Debug("process lang");
using (MemoryStream memStream = new MemoryStream(cdsData))
{
using (BinaryReader dis = new BinaryReader(memStream))
{
int entryCount = dis.ReadInt32();
byte[] freqData = dis.ReadBytes(1024);
HuffmanReader reader = new HuffmanReader(freqData);
List <int> keys = new List <int>(entryCount);
for (int i = 0; i < entryCount; i++)
{
int key = dis.ReadInt32();
int offset = Util.readUnsignedLeb128_X(dis.BaseStream);
keys.Add(key);
}
for (int i = 0; i < entryCount; i++)
{
if (progress != null)
{
progress.Invoke("english " + i + "/" + entryCount);
}
int compressedSize = Assets.RiftAssets.Util.readUnsignedLeb128_X(dis.BaseStream);
int uncompressedSize = Assets.RiftAssets.Util.readUnsignedLeb128_X(dis.BaseStream);
byte[] data = dis.ReadBytes(compressedSize);
byte[] dataOut = new byte[uncompressedSize];
dataOut = reader.read(data, data.Length, dataOut.Length);
LangEntry entry = new LangEntry();
entry.key = keys[i];
entry.cdata = dataOut;
stringMap[entry.key] = entry;
}
}
}
logger.Debug("done process lang");
progress.Invoke("done");
}
public void JustPrintTree()
{
string inputFile = @"TestFiles\Huff3Tests\simple4.in";
string[] args = new string[1];
args[0] = inputFile;
bool opened = Program.OpenFile(args, out BinaryReader Reader, out BinaryWriter Writer);
if (!opened)
{
return;
}
//create HuffmanReader, that is going to count occurences of bytes in input file
HuffmanReader HuffReader = new HuffmanReader(Reader);
bool NotEmpty = HuffReader.ReadFileUsingBuffer();
//Build Huffman tree
TreeBuilder Builder = new TreeBuilder(HuffReader.ProvideSymbolsWihtWeights());
Node Root = Builder.BuildHuffTree();
TreePrinter.PrintNiceTree(Root, 0);
Reader.Close();
Writer.Close();
}
public void ReadFileUsingBufferTest()
{
Stream s = File.OpenRead(@"TestFiles\Ins\ValidInput.bin");
BinaryReader br = new BinaryReader(s);
HuffmanReader reader = new HuffmanReader(br);
bool b = reader.ReadFileUsingBuffer();
Assert.IsTrue(b);
for (int i = 0; i < 256; i++)
{
if (reader.SymbolsWithWeights[i] > 0)
{
Console.WriteLine("{0}:{1}", i, reader.SymbolsWithWeights[i]);
}
}
Dictionary <byte, ulong> Test = new Dictionary <byte, ulong>();
Test[97] = 6;
Test[98] = 5;
Test[32] = 5;
Test[99] = 5;
Test[100] = 1;
Test[101] = 2;
Test[102] = 2;
Test[52] = 2;
foreach (var item in Test)
{
if (Test[item.Key] != reader.SymbolsWithWeights[item.Key])
{
Console.WriteLine($"key: {item.Key} test:{Test[item.Key]} reader:{reader.SymbolsWithWeights[item.Key]}");
Assert.Fail();
}
//reader.SymbolsWithWeights.Remove(item.Key);
}
}
public override void Decode(Stream input, Stream output)
{
using (var br = new BitReader(input, BitOrder.MsbFirst, 4, ByteOrder.LittleEndian))
{
var uncompressedSize = br.ReadInt32() >> 8;
br.ReadByte();
HuffmanReader.BuildTree(br);
SetupDisplacementTable(br, 7);
while (output.Length < uncompressedSize)
{
if (br.ReadBit() == 1)
{
var matchLength = 0;
int displacement;
var dispIndex = br.ReadBits <byte>(3);
if (dispIndex == 7)
{
// 8098E4C
byte readValue;
// Add lengthBitCount bit values as long as read values' LSB is set
// Seems to be a variable length value
do
{
readValue = br.ReadBits <byte>(4);
matchLength = (matchLength << 3) | (readValue >> 1);
} while ((readValue & 0x1) == 1);
if (br.ReadBit() == 1)
{
// 8098E64
dispIndex = br.ReadBits <byte>(3);
displacement = GetDisplacement(br, dispIndex);
matchLength = ((matchLength << 4) | br.ReadBits <byte>(4)) + 3;
// Goto 8098EA2
}
else
{
// 8098E88
matchLength++;
ReadHuffmanValues(br, output, matchLength, 1);
continue;
}
}
else
{
// 8098E32
displacement = GetDisplacement(br, dispIndex);
matchLength = br.ReadBits <byte>(4) + 3;
// Goto 8098EA2
}
// Label 8098EA2
ReadDisplacement(output, displacement, matchLength, 1);
}
else
{
// 8098E14
ReadHuffmanValues(br, output, 1, 1);
}
}
}
}
private static byte[] getData(byte[] compressedData, byte[] freq)
{
HuffmanReader reader = new HuffmanReader(freq);
return(getData(compressedData, reader));
}
public Enumerator(HuffmanReader <T> reader)
{
this.reader = reader;
Count = reader.reader.ReadInt32();
}
