public void CompressUncompressEmpty()
{
var compressed = CompressTool.Compress(new byte[0]);
Assert.IsTrue(compressed.Length > 0);
Assert.AreEqual(0, CompressTool.Uncompress(compressed).Length);
}
public override ImageFile.FileData OnDeserialize(byte[] data, int length)
{
var dataValid = data.SubArray(0, length);
var newData = CompressTool.Decompress(dataValid, CompressTool.CompressAlgorithm.Zstd);
return(ImageFile.FileData.Parser.ParseFrom(newData));
}
public override byte[] OnSerialize(ImageFile.FileData data)
{
var rawData = data.ToByteArray();
var newData = CompressTool.Compress(rawData, CompressTool.CompressAlgorithm.Zstd);
UnityEngine.Debug.LogFormat("Raw data size {0} KB, Compressed data size {1}",
rawData.Length / 1024, newData.Length > 1024?newData.Length / 1024 + "KB": newData.Length + "B");
return(newData);
}
public override void ThreadExtract7z(string filePath, string outPath, ExtractCallback callback)
{
CompressTool ct = new CompressTool();
ct.zipPath = filePath;
ct.tempPath = outPath;
ct.Extract();
callback(10, 10);
}
public override bool Extract7z(string filePath, string outPath)
{
CompressTool ct = new CompressTool();
ct.zipPath = filePath;
ct.tempPath = outPath + "res/";
ct.Extract();
return(true);
}
public void CompressUncompressTest()
{
var data = _dataCreator.GenerateTestData(1024 * 1024);
var compressed = CompressTool.Compress(data);
Assert.IsTrue(compressed.Length > 0);
Assert.IsTrue(data.SequenceEqual(CompressTool.Uncompress(compressed)));
}
public void CompressRange()
{
var input = Encoding.ASCII.GetBytes("ByeHelloBye");
var output = new byte[100];
var length = CompressTool.Compress(input, 3, 5, output, 10);
Assert.AreEqual("Hello",
Encoding.ASCII.GetString(CompressTool.Uncompress(output.Skip(10).Take(length).ToArray())));
}
/// <summary>
/// Resets this frame to contain compressed data.
/// </summary>
/// <param name="data">Input buffer containing uncompressed data that is compressed by this method before being stored in the frame.</param>
/// <param name="offset">Offset of uncompressed data in the input buffer.</param>
/// <param name="count">Size of uncompressed data in the input buffer. Maximum data size is 64KB.</param>
public void SetCompressed(byte[] data, int offset, int count)
{
CheckRange(data, offset, count);
CheckMaxFrameSize(count);
EnsureBuffer(CompressTool.GetMaxCompressedLength(count));
BufferUsage = CompressTool.Compress(data, offset, count, Buffer, 0);
DataLength = count;
Checksum = ComputeMasked(data, offset, count);
Type = SnappyFrameType.Compressed;
}
public void UncompressRange()
{
var howdy = Encoding.ASCII.GetBytes("Howdy");
var padded = howdy.Take(3).Concat(CompressTool.Compress(Encoding.ASCII.GetBytes("Hello"))).Concat(howdy.Skip(3)).ToArray();
var output = new byte[100];
var length = CompressTool.Uncompress(padded, 3, padded.Length - 5, output, 10);
Assert.AreEqual(5, length);
Assert.AreEqual("Hello", Encoding.ASCII.GetString(output.Skip(10).Take(5).ToArray()));
}
public void GetUncompressedLengthExceptions()
{
var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?");
var compressed = CompressTool.Compress(uncompressed);
var buffer = new byte[100];
TestException(() => { CompressTool.GetUncompressedLength(null, 0, 3); }, typeof(ArgumentNullException));
TestException(() => { CompressTool.GetUncompressedLength(compressed, -1, uncompressed.Length); }, typeof(ArgumentOutOfRangeException));
TestException(() => { CompressTool.GetUncompressedLength(compressed, 0, -1); }, typeof(ArgumentOutOfRangeException));
TestException(() => { CompressTool.GetUncompressedLength(compressed, compressed.Length - 2, 4); }, typeof(ArgumentOutOfRangeException));
TestException(() => { CompressTool.GetUncompressedLength(compressed, 0, 0); }, typeof(System.IO.IOException));
TestException(() => { CompressTool.GetUncompressedLength(compressed, compressed.Length, 0); }, typeof(System.IO.IOException));
}
private void DoUncompressionBenchmark(int size)
{
Benchmark.Run("Uncompressing", size, benchmark =>
{
var compressed = CompressTool.Compress(benchmark.Input);
var roundtrip = new byte[benchmark.Input.Length];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = CompressTool.Uncompress(compressed, 0, compressed.Length, roundtrip, 0);
}
benchmark.Stopwatch.Stop();
CollectionAssert.AreEqual(benchmark.Input, roundtrip);
});
}
/// <summary>
/// Create json file
/// </summary>
private void CreateJsonFile(string path, string savePath)
{
csFileName = Path.GetFileNameWithoutExtension(path);
fileName = csFileName + ".json";
//if (!cbIsDat.Checked)
// fileName = "__" + fileName;
bool showOutput = checkBoxShowOutput.Checked;
string strJson = ExcelToJsonStr(path, cbIsCsFile.Checked);
if (strJson != string.Empty && strJson != null)
{
savePath = savePath + "\\" + fileName;
string writeStr = string.Empty;
if (cbIsDat.Checked)
{
writeStr = CompressTool.Compress(strJson);
}
else
{
writeStr = strJson;
}
FileStream fs = new FileStream(savePath, FileMode.Create);
StreamWriter sw = new StreamWriter(fs);
sw.Write(writeStr);
sw.Close();
txtInfo.Text += "Generated: " + fileName + "\r\n";
if (showOutput)
{
textJsonInfo.Text += "Json: " + fileName + "\r\n" + strJson + "\r\n\r\n";
}
}
else
{
txtInfo.Text += fileName + " not generated ERROR\r\n";
textJsonInfo.Text += fileName + " not generated ERROR\r\n";
}
if (showOutput)
{
this.textJsonInfo.Select(this.textJsonInfo.TextLength, 0);
this.textJsonInfo.ScrollToCaret();
this.txtInfo.Select(this.textJsonInfo.TextLength, 0);
this.txtInfo.ScrollToCaret();
}
}
private void DoCompressionOneParaBenchmark(int size)
{
Benchmark.Run("Compressing", size, benchmark =>
{
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
var output = CompressTool.Compress(benchmark.Input);
length = output.Length;
}
benchmark.Stopwatch.Stop();
benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length);
});
}
private void DoCompressionBenchmark(int size)
{
Benchmark.Run("Compressing", size, benchmark =>
{
var output = new byte[CompressTool.GetMaxCompressedLength(benchmark.Input.Length)];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = CompressTool.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0);
}
benchmark.Stopwatch.Stop();
var roundtrip = new byte[benchmark.Input.Length];
var roundtripLength = CompressTool.Uncompress(output, 0, length, roundtrip, 0);
Assert.IsTrue(benchmark.Input.SequenceEqual(roundtrip.Take(roundtripLength)));
benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length);
});
}
private Task DoAsyncUncompressionOneParaBenchmark(int size)
{
return(Task.Factory.StartNew(() =>
{
Benchmark.Run("Uncompressing", size, benchmark =>
{
var compressed = CompressTool.Compress(benchmark.Input);
var roundtrip = new byte[benchmark.Input.Length];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
roundtrip = CompressTool.Uncompress(compressed);
length = roundtrip.Length;
}
benchmark.Stopwatch.Stop();
CollectionAssert.AreEqual(benchmark.Input, roundtrip);
});
}));
}
private Task DoAsyncCompressionOneParaBenchmark(int size)
{
return(Task.Factory.StartNew(() =>
{
Benchmark.Run("Compressing", size, benchmark =>
{
var output = new byte[CompressTool.GetMaxCompressedLength(benchmark.Input.Length)];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
output = CompressTool.Compress(benchmark.Input);
length = output.Length;
}
benchmark.Stopwatch.Stop();
var roundtrip = CompressTool.Uncompress(output);
Assert.IsTrue(benchmark.Input.SequenceEqual(roundtrip));
benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length);
});
}));
}
/// <summary>
/// Retrieves data from the frame. Data is uncompressed before being stored in the output buffer.
/// CRC of the data is checked and CRC failure results in exception.
/// </summary>
/// <param name="buffer">Output buffer where uncompressed data is stored. Buffer length minus offset must be at least DataLength bytes.</param>
/// <param name="offset">Offset into the output buffer where uncompressed data will be stored.</param>
public void GetData(byte[] buffer, int offset)
{
if (Type == SnappyFrameType.Compressed)
{
var count = CompressTool.Uncompress(Buffer, 0, BufferUsage, buffer, offset);
if (ComputeMasked(buffer, offset, count) != Checksum)
{
throw new InvalidDataException();
}
}
else if (Type == SnappyFrameType.Uncompressed)
{
if (ComputeMasked(Buffer, offset, BufferUsage) != Checksum)
{
throw new InvalidDataException();
}
Array.Copy(Buffer, 0, buffer, offset, BufferUsage);
}
else
{
throw new InvalidOperationException();
}
}
/// <summary>
/// Retrieves Snappy frame from underlying stream. Retrieved frame data is stored in properties of this object.
/// Return value indicates end of stream. Exceptions indicate data integrity errors and underlying stream errors.
/// </summary>
/// <param name="stream">Underlying stream that will be read by this method.</param>
/// <returns>
/// True if frame was successfully retrieved. False if there are no more frames in the stream, i.e. the end of stream has been reached.
/// Note that reaching the end of stream in the middle of the frame is considered an error and causes exception instead.
/// </returns>
public bool Read(Stream stream)
{
try
{
var headerRead = stream.Read(WordBuffer, 0, 4);
if (headerRead == 0)
{
return(false);
}
EnsureRead(stream, WordBuffer, headerRead, 4 - headerRead);
Type = (SnappyFrameType)WordBuffer[0];
int length = WordBuffer[1] + ((int)WordBuffer[2] << 8) + ((int)WordBuffer[3] << 16);
if (Type == SnappyFrameType.Compressed || Type == SnappyFrameType.Uncompressed)
{
if (length < 4)
{
throw new InvalidDataException();
}
EnsureRead(stream, WordBuffer, 0, 4);
Checksum = (uint)WordBuffer[0] + ((uint)WordBuffer[1] << 8) + ((uint)WordBuffer[2] << 16) + ((uint)WordBuffer[3] << 24);
BufferUsage = length - 4;
if (BufferUsage > MaxBufferUsage)
{
throw new InvalidDataException();
}
EnsureBuffer(BufferUsage);
EnsureRead(stream, Buffer, 0, BufferUsage);
DataLength = Type == SnappyFrameType.Uncompressed ? BufferUsage : CompressTool.GetUncompressedLength(Buffer, 0, BufferUsage);
if (DataLength > MaxFrameSize)
{
throw new InvalidDataException();
}
}
else if (Type == SnappyFrameType.Padding || (byte)Type >= (byte)SnappyFrameType.SkippableFirst && (byte)Type <= (byte)SnappyFrameType.SkippableLast)
{
DataLength = length;
BufferUsage = 0;
Checksum = 0;
SkipBytes(stream, length);
}
else if (Type == SnappyFrameType.StreamIdentifier)
{
if (length != 6)
{
throw new InvalidOperationException();
}
DataLength = 6;
BufferUsage = 0;
Checksum = 0;
EnsureBuffer(6);
EnsureRead(stream, Buffer, 0, 6);
if (!Utils.BuffersEqual(Buffer, StreamIdentifier, 6))
{
throw new InvalidDataException();
}
}
else
{
throw new InvalidDataException();
}
return(true);
}
catch
{
SetPadding(0);
throw;
}
}
public void CompressUncompressSimple()
{
Assert.AreEqual("Hello",
Encoding.ASCII.GetString(
CompressTool.Uncompress(CompressTool.Compress(Encoding.ASCII.GetBytes("Hello")))));
}
static void Main(string[] args)
{
if (!Config.Load())
{
Console.WriteLine("请添加配置文件！");
return;
}
// TODO Dll path 改成xbuff_config的
// 验证反射对没对
Assembly asm = Assembly.LoadFrom("xbuffer_config.dll");
if (asm == null)
{
Console.WriteLine("dll不存在！");
return;
}
configAssembly = asm;
var csvPath = Config.Get("csv_path");
var csvFileList = Directory.GetFiles(csvPath);
Type t = asm.GetType("xbuffer." + Config.Get("mgr_class_name"));
//ConfigManager obj = new ConfigManager();
//Type t = obj.GetType();
object obj = asm.CreateInstance(t.FullName);
for (int i = 0; i < csvFileList.Length; ++i)
{
var csvFile = csvFileList[i];
if (csvFile.EndsWith(".csv"))
{
Console.WriteLine("--- Insert Bytes Data ---" + i + "---" + Path.GetFileName(csvFile));
CsvReader reader = CsvReader.ReadFile(csvFile);
var classInfo = CsvParser.Parse(reader.name, reader.table);
FillFieldData(t, obj, classInfo, reader.table);
}
}
xbuffer.XSteam stream = new xbuffer.XSteam(1, 1024 * 1024 * 100);
Type bufferT = asm.GetType("xbuffer." + Config.Get("mgr_class_name") + "Buffer");
object[] arg = new object[2] {
obj, stream
};
var serializeMethod = bufferT.GetMethod("serialize", BindingFlags.Static | BindingFlags.Public);
if (serializeMethod != null)
{
serializeMethod.Invoke(null, arg);
var buffer = stream.getBytes();
buffer = CompressTool.CompressBytes(buffer); //压缩一下
File.WriteAllBytes(Config.Get("data_out_path") + "/" + "ConfigData.bytes", buffer);
}
else
{
throw new Exception("找不到方法 ===> xbuffer." + Config.Get("mgr_class_name") + "Buffer.serialize");
}
Console.WriteLine("Finish!!!");
}
public void CompressExceptions()
{
var input = new byte[100];
var output = new byte[100];
try
{
CompressTool.Compress(null);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentNullException));
}
try
{
CompressTool.Compress(null, 0, 3, output, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentNullException));
}
try
{
CompressTool.Compress(input, 0, 3, null, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentNullException));
}
try
{
CompressTool.Compress(input, -1, 3, output, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, -1, 3, output, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 0, -1, output, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 90, 20, output, 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 0, 3, output, -1);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 0, 3, output, 100);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 0, 3, output, 101);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
try
{
CompressTool.Compress(input, 0, 100, new byte[3], 0);
Assert.Fail("shouldn't reach there");
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException));
}
}
public override Data OnDeserialize(byte[] data, int length)
{
var remote = CompressTool.Decompress(data);
return(Serilization.ByteArrayToObject <Data>(remote));
}
public override byte[] OnSerialize(Data data)
{
var raw = Serilization.ObjectToByteArray(data);
return(CompressTool.Compress(raw));
}
