public void CompressUncompressEmpty() { var compressed = CompressTool.Compress(new byte[0]); Assert.IsTrue(compressed.Length > 0); Assert.AreEqual(0, CompressTool.Uncompress(compressed).Length); }
public override ImageFile.FileData OnDeserialize(byte[] data, int length) { var dataValid = data.SubArray(0, length); var newData = CompressTool.Decompress(dataValid, CompressTool.CompressAlgorithm.Zstd); return(ImageFile.FileData.Parser.ParseFrom(newData)); }
public override byte[] OnSerialize(ImageFile.FileData data) { var rawData = data.ToByteArray(); var newData = CompressTool.Compress(rawData, CompressTool.CompressAlgorithm.Zstd); UnityEngine.Debug.LogFormat("Raw data size {0} KB, Compressed data size {1}", rawData.Length / 1024, newData.Length > 1024?newData.Length / 1024 + "KB": newData.Length + "B"); return(newData); }
public override void ThreadExtract7z(string filePath, string outPath, ExtractCallback callback) { CompressTool ct = new CompressTool(); ct.zipPath = filePath; ct.tempPath = outPath; ct.Extract(); callback(10, 10); }
public override bool Extract7z(string filePath, string outPath) { CompressTool ct = new CompressTool(); ct.zipPath = filePath; ct.tempPath = outPath + "res/"; ct.Extract(); return(true); }
public void CompressUncompressTest() { var data = _dataCreator.GenerateTestData(1024 * 1024); var compressed = CompressTool.Compress(data); Assert.IsTrue(compressed.Length > 0); Assert.IsTrue(data.SequenceEqual(CompressTool.Uncompress(compressed))); }
public void CompressRange() { var input = Encoding.ASCII.GetBytes("ByeHelloBye"); var output = new byte[100]; var length = CompressTool.Compress(input, 3, 5, output, 10); Assert.AreEqual("Hello", Encoding.ASCII.GetString(CompressTool.Uncompress(output.Skip(10).Take(length).ToArray()))); }
/// <summary> /// Resets this frame to contain compressed data. /// </summary> /// <param name="data">Input buffer containing uncompressed data that is compressed by this method before being stored in the frame.</param> /// <param name="offset">Offset of uncompressed data in the input buffer.</param> /// <param name="count">Size of uncompressed data in the input buffer. Maximum data size is 64KB.</param> public void SetCompressed(byte[] data, int offset, int count) { CheckRange(data, offset, count); CheckMaxFrameSize(count); EnsureBuffer(CompressTool.GetMaxCompressedLength(count)); BufferUsage = CompressTool.Compress(data, offset, count, Buffer, 0); DataLength = count; Checksum = ComputeMasked(data, offset, count); Type = SnappyFrameType.Compressed; }
public void UncompressRange() { var howdy = Encoding.ASCII.GetBytes("Howdy"); var padded = howdy.Take(3).Concat(CompressTool.Compress(Encoding.ASCII.GetBytes("Hello"))).Concat(howdy.Skip(3)).ToArray(); var output = new byte[100]; var length = CompressTool.Uncompress(padded, 3, padded.Length - 5, output, 10); Assert.AreEqual(5, length); Assert.AreEqual("Hello", Encoding.ASCII.GetString(output.Skip(10).Take(5).ToArray())); }
public void GetUncompressedLengthExceptions() { var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); var compressed = CompressTool.Compress(uncompressed); var buffer = new byte[100]; TestException(() => { CompressTool.GetUncompressedLength(null, 0, 3); }, typeof(ArgumentNullException)); TestException(() => { CompressTool.GetUncompressedLength(compressed, -1, uncompressed.Length); }, typeof(ArgumentOutOfRangeException)); TestException(() => { CompressTool.GetUncompressedLength(compressed, 0, -1); }, typeof(ArgumentOutOfRangeException)); TestException(() => { CompressTool.GetUncompressedLength(compressed, compressed.Length - 2, 4); }, typeof(ArgumentOutOfRangeException)); TestException(() => { CompressTool.GetUncompressedLength(compressed, 0, 0); }, typeof(System.IO.IOException)); TestException(() => { CompressTool.GetUncompressedLength(compressed, compressed.Length, 0); }, typeof(System.IO.IOException)); }
private void DoUncompressionBenchmark(int size) { Benchmark.Run("Uncompressing", size, benchmark => { var compressed = CompressTool.Compress(benchmark.Input); var roundtrip = new byte[benchmark.Input.Length]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { length = CompressTool.Uncompress(compressed, 0, compressed.Length, roundtrip, 0); } benchmark.Stopwatch.Stop(); CollectionAssert.AreEqual(benchmark.Input, roundtrip); }); }
/// <summary> /// Create json file /// </summary> private void CreateJsonFile(string path, string savePath) { csFileName = Path.GetFileNameWithoutExtension(path); fileName = csFileName + ".json"; //if (!cbIsDat.Checked) // fileName = "__" + fileName; bool showOutput = checkBoxShowOutput.Checked; string strJson = ExcelToJsonStr(path, cbIsCsFile.Checked); if (strJson != string.Empty && strJson != null) { savePath = savePath + "\\" + fileName; string writeStr = string.Empty; if (cbIsDat.Checked) { writeStr = CompressTool.Compress(strJson); } else { writeStr = strJson; } FileStream fs = new FileStream(savePath, FileMode.Create); StreamWriter sw = new StreamWriter(fs); sw.Write(writeStr); sw.Close(); txtInfo.Text += "Generated: " + fileName + "\r\n"; if (showOutput) { textJsonInfo.Text += "Json: " + fileName + "\r\n" + strJson + "\r\n\r\n"; } } else { txtInfo.Text += fileName + " not generated ERROR\r\n"; textJsonInfo.Text += fileName + " not generated ERROR\r\n"; } if (showOutput) { this.textJsonInfo.Select(this.textJsonInfo.TextLength, 0); this.textJsonInfo.ScrollToCaret(); this.txtInfo.Select(this.textJsonInfo.TextLength, 0); this.txtInfo.ScrollToCaret(); } }
private void DoCompressionOneParaBenchmark(int size) { Benchmark.Run("Compressing", size, benchmark => { int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { var output = CompressTool.Compress(benchmark.Input); length = output.Length; } benchmark.Stopwatch.Stop(); benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length); }); }
private void DoCompressionBenchmark(int size) { Benchmark.Run("Compressing", size, benchmark => { var output = new byte[CompressTool.GetMaxCompressedLength(benchmark.Input.Length)]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { length = CompressTool.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0); } benchmark.Stopwatch.Stop(); var roundtrip = new byte[benchmark.Input.Length]; var roundtripLength = CompressTool.Uncompress(output, 0, length, roundtrip, 0); Assert.IsTrue(benchmark.Input.SequenceEqual(roundtrip.Take(roundtripLength))); benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length); }); }
private Task DoAsyncUncompressionOneParaBenchmark(int size) { return(Task.Factory.StartNew(() => { Benchmark.Run("Uncompressing", size, benchmark => { var compressed = CompressTool.Compress(benchmark.Input); var roundtrip = new byte[benchmark.Input.Length]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { roundtrip = CompressTool.Uncompress(compressed); length = roundtrip.Length; } benchmark.Stopwatch.Stop(); CollectionAssert.AreEqual(benchmark.Input, roundtrip); }); })); }
private Task DoAsyncCompressionOneParaBenchmark(int size) { return(Task.Factory.StartNew(() => { Benchmark.Run("Compressing", size, benchmark => { var output = new byte[CompressTool.GetMaxCompressedLength(benchmark.Input.Length)]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { output = CompressTool.Compress(benchmark.Input); length = output.Length; } benchmark.Stopwatch.Stop(); var roundtrip = CompressTool.Uncompress(output); Assert.IsTrue(benchmark.Input.SequenceEqual(roundtrip)); benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length); }); })); }
/// <summary> /// Retrieves data from the frame. Data is uncompressed before being stored in the output buffer. /// CRC of the data is checked and CRC failure results in exception. /// </summary> /// <param name="buffer">Output buffer where uncompressed data is stored. Buffer length minus offset must be at least DataLength bytes.</param> /// <param name="offset">Offset into the output buffer where uncompressed data will be stored.</param> public void GetData(byte[] buffer, int offset) { if (Type == SnappyFrameType.Compressed) { var count = CompressTool.Uncompress(Buffer, 0, BufferUsage, buffer, offset); if (ComputeMasked(buffer, offset, count) != Checksum) { throw new InvalidDataException(); } } else if (Type == SnappyFrameType.Uncompressed) { if (ComputeMasked(Buffer, offset, BufferUsage) != Checksum) { throw new InvalidDataException(); } Array.Copy(Buffer, 0, buffer, offset, BufferUsage); } else { throw new InvalidOperationException(); } }
/// <summary> /// Retrieves Snappy frame from underlying stream. Retrieved frame data is stored in properties of this object. /// Return value indicates end of stream. Exceptions indicate data integrity errors and underlying stream errors. /// </summary> /// <param name="stream">Underlying stream that will be read by this method.</param> /// <returns> /// True if frame was successfully retrieved. False if there are no more frames in the stream, i.e. the end of stream has been reached. /// Note that reaching the end of stream in the middle of the frame is considered an error and causes exception instead. /// </returns> public bool Read(Stream stream) { try { var headerRead = stream.Read(WordBuffer, 0, 4); if (headerRead == 0) { return(false); } EnsureRead(stream, WordBuffer, headerRead, 4 - headerRead); Type = (SnappyFrameType)WordBuffer[0]; int length = WordBuffer[1] + ((int)WordBuffer[2] << 8) + ((int)WordBuffer[3] << 16); if (Type == SnappyFrameType.Compressed || Type == SnappyFrameType.Uncompressed) { if (length < 4) { throw new InvalidDataException(); } EnsureRead(stream, WordBuffer, 0, 4); Checksum = (uint)WordBuffer[0] + ((uint)WordBuffer[1] << 8) + ((uint)WordBuffer[2] << 16) + ((uint)WordBuffer[3] << 24); BufferUsage = length - 4; if (BufferUsage > MaxBufferUsage) { throw new InvalidDataException(); } EnsureBuffer(BufferUsage); EnsureRead(stream, Buffer, 0, BufferUsage); DataLength = Type == SnappyFrameType.Uncompressed ? BufferUsage : CompressTool.GetUncompressedLength(Buffer, 0, BufferUsage); if (DataLength > MaxFrameSize) { throw new InvalidDataException(); } } else if (Type == SnappyFrameType.Padding || (byte)Type >= (byte)SnappyFrameType.SkippableFirst && (byte)Type <= (byte)SnappyFrameType.SkippableLast) { DataLength = length; BufferUsage = 0; Checksum = 0; SkipBytes(stream, length); } else if (Type == SnappyFrameType.StreamIdentifier) { if (length != 6) { throw new InvalidOperationException(); } DataLength = 6; BufferUsage = 0; Checksum = 0; EnsureBuffer(6); EnsureRead(stream, Buffer, 0, 6); if (!Utils.BuffersEqual(Buffer, StreamIdentifier, 6)) { throw new InvalidDataException(); } } else { throw new InvalidDataException(); } return(true); } catch { SetPadding(0); throw; } }
public void CompressUncompressSimple() { Assert.AreEqual("Hello", Encoding.ASCII.GetString( CompressTool.Uncompress(CompressTool.Compress(Encoding.ASCII.GetBytes("Hello"))))); }
static void Main(string[] args) { if (!Config.Load()) { Console.WriteLine("请添加配置文件!"); return; } // TODO Dll path 改成xbuff_config的 // 验证反射对没对 Assembly asm = Assembly.LoadFrom("xbuffer_config.dll"); if (asm == null) { Console.WriteLine("dll不存在!"); return; } configAssembly = asm; var csvPath = Config.Get("csv_path"); var csvFileList = Directory.GetFiles(csvPath); Type t = asm.GetType("xbuffer." + Config.Get("mgr_class_name")); //ConfigManager obj = new ConfigManager(); //Type t = obj.GetType(); object obj = asm.CreateInstance(t.FullName); for (int i = 0; i < csvFileList.Length; ++i) { var csvFile = csvFileList[i]; if (csvFile.EndsWith(".csv")) { Console.WriteLine("--- Insert Bytes Data ---" + i + "---" + Path.GetFileName(csvFile)); CsvReader reader = CsvReader.ReadFile(csvFile); var classInfo = CsvParser.Parse(reader.name, reader.table); FillFieldData(t, obj, classInfo, reader.table); } } xbuffer.XSteam stream = new xbuffer.XSteam(1, 1024 * 1024 * 100); Type bufferT = asm.GetType("xbuffer." + Config.Get("mgr_class_name") + "Buffer"); object[] arg = new object[2] { obj, stream }; var serializeMethod = bufferT.GetMethod("serialize", BindingFlags.Static | BindingFlags.Public); if (serializeMethod != null) { serializeMethod.Invoke(null, arg); var buffer = stream.getBytes(); buffer = CompressTool.CompressBytes(buffer); //压缩一下 File.WriteAllBytes(Config.Get("data_out_path") + "/" + "ConfigData.bytes", buffer); } else { throw new Exception("找不到方法 ===> xbuffer." + Config.Get("mgr_class_name") + "Buffer.serialize"); } Console.WriteLine("Finish!!!"); }
public void CompressExceptions() { var input = new byte[100]; var output = new byte[100]; try { CompressTool.Compress(null); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentNullException)); } try { CompressTool.Compress(null, 0, 3, output, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentNullException)); } try { CompressTool.Compress(input, 0, 3, null, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentNullException)); } try { CompressTool.Compress(input, -1, 3, output, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, -1, 3, output, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 0, -1, output, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 90, 20, output, 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 0, 3, output, -1); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 0, 3, output, 100); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 0, 3, output, 101); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } try { CompressTool.Compress(input, 0, 100, new byte[3], 0); Assert.Fail("shouldn't reach there"); } catch (Exception ex) { Assert.IsInstanceOfType(ex, typeof(ArgumentOutOfRangeException)); } }
public override Data OnDeserialize(byte[] data, int length) { var remote = CompressTool.Decompress(data); return(Serilization.ByteArrayToObject <Data>(remote)); }
public override byte[] OnSerialize(Data data) { var raw = Serilization.ObjectToByteArray(data); return(CompressTool.Compress(raw)); }