public void CompressDynamicResolution <T>(T[] input) { int typeSize = 0; try { typeSize = Marshal.SizeOf(input[0]); } catch { if (typeof(T) == typeof(DateTime)) { typeSize = 8; } else if (typeof(T) == typeof(string)) { typeSize = (input[input.Length - 1] as string).Length; } else if (typeof(T) == typeof(ComplexObject)) { typeSize = ComplexObject.Create().TextValue.Length + 16000 + 8000; } } byte[] compr = new byte[0]; var sw = new Stopwatch(); sw.Start(); var rounds = 500; for (int i = 0; i < rounds; i++) { compr = Serializer.Serialize(input); } sw.Stop(); var cspeed = 1000.0 * ((double)(input.Length * typeSize * rounds) / (1024.0 * 1024)) / ((double)sw.ElapsedMilliseconds); // MB/s var celapsed = sw.ElapsedMilliseconds; var saving = -Math.Round(100.0 * (((double)compr.Length) / ((double)(input.Length * typeSize)) - 1.0), 1); Console.WriteLine(" cbytes: " + compr.Length + " saving: " + saving); sw.Restart(); T[] decompr = new T[0]; for (int i = 0; i < rounds; i++) { decompr = Serializer.Deserialize <T[]>(compr); } sw.Stop(); var delapsed = sw.ElapsedMilliseconds; var dspeed = 1000.0 * ((double)(input.Length * typeSize * rounds) / (1024.0 * 1024)) / ((double)sw.ElapsedMilliseconds); // MB/s Assert.AreEqual(decompr.Length, input.Length); Assert.IsTrue(decompr.SequenceEqual(input)); Console.WriteLine(" >: " + Math.Round(cspeed, 2) + "MB/s; save/sec: " + Math.Round((saving * input.Length * typeSize * rounds / (100 * 1024 * 1024)) / (celapsed / 1000.0), 2)); Console.WriteLine(" <: " + Math.Round(dspeed, 2) + "MB/s; save/sec: " + Math.Round((saving * input.Length * typeSize * rounds / (100 * 1024 * 1024)) / (delapsed / 1000.0), 2)); }
public void Init() { var previous = 100.0; for (var i = 0; i < _big; i++) { var val = Math.Round(previous * (1 + (_rng.NextDouble() * 0.004 - 0.001)), 2); if (i < _small) { _doublesSmall[i] = val; _decsSmall[i] = (decimal)val; _longsSmall[i] = i; _datesSmall[i] = DateTime.UtcNow.Date.AddDays(i); _tickSmall[i] = new Tick(DateTime.UtcNow.Date.AddSeconds(i), new Price((double)i, 5), i); if (i < 10) { _complexSmall[i] = ComplexObject.Create(); } _stringSmall[i] = _rng.NextDouble().ToString(); } _doublesBig[i] = val; _decsBig[i] = (decimal)val; _longsBig[i] = i; _datesBig[i] = DateTime.UtcNow.Date.AddDays(i); _tickBig[i] = new Tick(DateTime.UtcNow.Date.AddSeconds(i), new Price((double)i, 5), i); var dt = DateTimeOffset.Now; previous = val; } unsafe { Console.WriteLine($"Size of Tick: {Marshal.SizeOf(typeof(Tick))}"); Console.WriteLine($"Size of Price: {Marshal.SizeOf(typeof(Price))}"); Console.WriteLine($"Size of OHLC: {Marshal.SizeOf(typeof(OHLC))}"); fixed(Tick *destPtr = &_tickSmall[0]) { Console.WriteLine("Ticks pointer: " + ((IntPtr)destPtr)); } } }
public void CouldCompressAndDecompressComplexObject() { bool write = true; var sw = new Stopwatch(); sw.Start(); for (int rounds = 0; rounds < 1000; rounds++) { var complexObj = ComplexObject.Create(); var bytes = Serializer.Serialize(complexObj); if (write) { Console.WriteLine("Uncompressed size: " + (complexObj.TextValue.Length * 2 + 8000 + 16000)); Console.WriteLine("Compressed size: " + bytes.Length); write = false; } var complexObj2 = Serializer.Deserialize <ComplexObject>(bytes); Assert.IsTrue(complexObj.IntArray.SequenceEqual(complexObj2.IntArray)); Assert.AreEqual(complexObj.TextValue, complexObj2.TextValue); } sw.Stop(); Console.WriteLine("Elapsed: " + sw.ElapsedMilliseconds); // scope, null map { var complexObj = ComplexObject.Create(); complexObj.SortedMap = null; // not root, JSON.NET deals with it var bytes = Serializer.Serialize(complexObj); if (write) { Console.WriteLine("Uncompressed size: " + (complexObj.TextValue.Length * 2 + 8000 + 16000)); Console.WriteLine("Compressed size: " + bytes.Length); write = false; } var complexObj2 = Serializer.Deserialize <ComplexObject>(bytes); Assert.IsTrue(complexObj.IntArray.SequenceEqual(complexObj2.IntArray)); Assert.AreEqual(complexObj.TextValue, complexObj2.TextValue); } // scope, null int array { var complexObj = ComplexObject.Create(); complexObj.SortedMap = null; complexObj.IntArray = null; complexObj.TextValue = ""; var bytes = Serializer.Serialize(complexObj); if (write) { Console.WriteLine("Uncompressed size: " + (complexObj.TextValue.Length * 2 + 8000 + 16000)); Console.WriteLine("Compressed size: " + bytes.Length); write = false; } var complexObj2 = Serializer.Deserialize <ComplexObject>(bytes); Assert.IsTrue(complexObj2.IntArray == null); Assert.AreEqual(complexObj.TextValue, complexObj2.TextValue); } //scope, empty int array { var complexObj = ComplexObject.Create(); complexObj.SortedMap = null; complexObj.IntArray = new long[0]; complexObj.TextValue = ""; var bytes = Serializer.Serialize(complexObj); if (write) { Console.WriteLine("Uncompressed size: " + (complexObj.TextValue.Length * 2 + 8000 + 16000)); Console.WriteLine("Compressed size: " + bytes.Length); write = false; } var complexObj2 = Serializer.Deserialize <ComplexObject>(bytes); Assert.IsTrue(complexObj.IntArray.SequenceEqual(complexObj2.IntArray)); Assert.AreEqual(complexObj.TextValue, complexObj2.TextValue); } }
internal void Compress <T>(T[] input, CompressionMethod method, bool shuffle) { int typeSize = 0; try { typeSize = Marshal.SizeOf(input[0]); } catch { if (typeof(T) == typeof(DateTime)) { typeSize = 8; } else if (typeof(T) == typeof(string)) { typeSize = (input[input.Length - 1] as string).Length; } else if (typeof(T) == typeof(ComplexObject)) { typeSize = ComplexObject.Create().TextValue.Length + 16000 + 8000; } } Console.Write(" m: " + method); Console.WriteLine(" s: " + shuffle); Console.WriteLine(" nbytes: " + input.Length * typeSize); byte[] compr = new byte[0]; var sw = new Stopwatch(); sw.Start(); var rounds = 500; for (int i = 0; i < rounds; i++) { compr = Serializer.CompressArray <T>(input, level: 9, shuffle: shuffle, method: method, diff: true); } sw.Stop(); //var js = new SpreadsJsonSerializer(); //var json = js.SerializeToJson(input); //Console.WriteLine(json); //var fromJson = js.DeserializeFromJson<T[]>(@"[""0"", ""1""]"); var cspeed = 1000.0 * ((double)(input.Length * typeSize * rounds) / (1024.0 * 1024)) / ((double)sw.ElapsedMilliseconds); // MB/s var celapsed = sw.ElapsedMilliseconds; var saving = -Math.Round(100.0 * (((double)compr.Length) / ((double)(input.Length * typeSize)) - 1.0), 1); Console.WriteLine(" cbytes: " + compr.Length + " saving: " + saving); sw.Restart(); T[] decompr = new T[0]; for (int i = 0; i < rounds; i++) { decompr = Serializer.DecompressArray <T>(compr, diff: true); } sw.Stop(); var delapsed = sw.ElapsedMilliseconds; var dspeed = 1000.0 * ((double)(input.Length * typeSize * rounds) / (1024.0 * 1024)) / ((double)sw.ElapsedMilliseconds); // MB/s Assert.AreEqual(decompr.Length, input.Length); for (int i = 0; i < decompr.Length; i++) { if (!decompr[i].Equals(input[i])) { Console.WriteLine("In: " + input[i] + "; out: " + decompr[i]); } } //Assert.IsTrue(decompr.SequenceEqual(input)); Console.WriteLine(" >: " + Math.Round(cspeed, 2) + "MB/s; save/sec: " + Math.Round((saving * input.Length * typeSize * rounds / (100 * 1024 * 1024)) / (celapsed / 1000.0), 2)); Console.WriteLine(" <: " + Math.Round(dspeed, 2) + "MB/s; save/sec: " + Math.Round((saving * input.Length * typeSize * rounds / (100 * 1024 * 1024)) / (delapsed / 1000.0), 2)); }