public unsafe void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
        {
            var original = new byte[length];

            Lorem.Fill(original, 0, length);

            var pickled   = LZ4Pickler.Pickle(original, level);
            var unpickled = LZ4Pickler.Unpickle(pickled);

            Tools.SameBytes(original, unpickled);

            // reiterating same test, but with different overloads
            fixed(byte *p = original)
            pickled = LZ4Pickler.Pickle(p, original.Length, level);

            fixed(byte *p = pickled)
            unpickled = LZ4Pickler.Unpickle(p, pickled.Length);

            Tools.SameBytes(original, unpickled);

            // reiterating same test, but with offset
            var copy = new byte[pickled.Length + 37];

            Array.Copy(pickled, 0, copy, 37, pickled.Length);
            unpickled = LZ4Pickler.Unpickle(copy, 37, pickled.Length);
            Tools.SameBytes(original, unpickled);

            unpickled.AsSpan().Fill(0);
            LZ4Pickler.Unpickle(pickled.AsSpan(), unpickled.AsSpan());
            Tools.SameBytes(original, unpickled);
        }
示例#2
0
        public byte[] UnCompress(byte[] compressedProto, int offset, int length, out int outLength)
        {
            var unwrapped = LZ4Pickler.Unpickle(compressedProto, offset, length);

            outLength = unwrapped.Length;
            return(unwrapped);
        }
示例#3
0
        public object FromStream(ref BytePacker packer, IDecrypter decrypter)
        {
            BytePacker p = packer;

            if (Compression == Compression.LZ4)
            {
                byte[] encoded = (byte[])byteArrayConverter.Deserialize(p);
                byte[] data    = LZ4Pickler.Unpickle(encoded);

                if (decrypter != null)
                {
                    data = decrypter.Decrypt(data);
                }

                p = new BytePacker(data);
            }
            else if (decrypter != null)
            {
                try
                {
                    byte[] data = (byte[])encArrayConverter.Deserialize(p);
                    data = decrypter.Decrypt(data);
                    p    = new BytePacker(data);
                }
                catch (Exception e)
                {
                    Util.Log("FromStream:" + e.Message);
                }
            }

            return(converter.Deserialize(p));
        }
        /// <summary>
        /// Returns the decompressed string from the bytes. This methods can only be called once.
        /// </summary>
        /// <returns></returns>
        /// <exception cref="InvalidOperationException">Throws if this is called more than once</exception>
        public string DecompressString()
        {
            if (_str != null)
            {
                return(_str);
            }

            lock (_locker)
            {
                if (_str != null)
                {
                    // double check
                    return(_str);
                }

                if (_bytes == null)
                {
                    throw new InvalidOperationException("Bytes have already been cleared");
                }

                _str   = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes));
                _bytes = null;
            }
            return(_str);
        }
示例#5
0
        public static byte[] Unzip(byte[] data, CompressionMethod method = CompressionMethod.Lz4)
        {
            if (method == CompressionMethod.Lz4)
            {
                return(LZ4Pickler.Unpickle(data));
            }

            return(null);
        }
        public void PickleAndUnpickle(string filename)
        {
            var original = File.ReadAllBytes(Tools.FindFile(filename));

            var compressed   = LZ4Pickler.Pickle(original);
            var decompressed = LZ4Pickler.Unpickle(compressed);

            Tools.SameBytes(original, decompressed);
        }
        public void PickleEntropy(int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
        {
            var original = new byte[length];

            new Random(seed).NextBytes(original);

            var pickled   = LZ4Pickler.Pickle(original, level);
            var unpickled = LZ4Pickler.Unpickle(pickled);

            Tools.SameBytes(original, unpickled);
        }
示例#8
0
        public void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
        {
            var original = new byte[length];

            Lorem.Fill(original, 0, length);

            var pickled   = LZ4Pickler.Pickle(original, level);
            var unpickled = LZ4Pickler.Unpickle(pickled);

            Tools.SameBytes(original, unpickled);
        }
示例#9
0
 public object FromStream(ref BytePacker packer)
 {
     if (Compression == Compression.LZ4)
     {
         byte[]     encoded   = (byte[])lz4converter.Deserialize(packer);
         byte[]     data      = LZ4Pickler.Unpickle(encoded);
         BytePacker lz4packer = new BytePacker(data);
         return(converter.Deserialize(lz4packer));
     }
     else
     {
         return(converter.Deserialize(packer));
     }
 }
示例#10
0
        /// <summary>
        /// Deserializing binary data to a new instance. Decompression is using LZ4 pickle self-contained format.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="objectData">Compressed data expected</param>
        /// <returns></returns>
        public T Unpickle <T>(Stream objectData)
        {
            int length = 0;
            int offset = 0;

            byte[]       compressedBuff = null;
            MemoryStream mem;

            using (mem = new MemoryStream())
            {
                if (objectData is MemoryStream outMem)
                {
#if NETCOREAPP || NETSTANDARD
                    if (outMem.TryGetBuffer(out var arraySegment))
                    {
                        compressedBuff = arraySegment.Array;
                        length         = arraySegment.Count;
                        offset         = arraySegment.Offset;
                    }
#else
                    try
                    {
                        compressedBuff = outMem.GetBuffer();
                        length         = (int)outMem.Length;
                        offset         = (int)outMem.Position;
                    }
                    catch (UnauthorizedAccessException)
                    {
                        // eat the error
                    }
#endif
                }

                if (compressedBuff == null)
                {
                    objectData.CopyTo(mem);
                    compressedBuff = mem.GetBuffer();
                    length         = (int)mem.Length;
                    offset         = 0;
                }

                var serializedBuff = LZ4Pickler.Unpickle(compressedBuff, offset, length);

                mem.Dispose();
                mem = new MemoryStream(serializedBuff);

                return(_serializer.Deserialize <T>(mem));
            }
        }
        public void PicklingSpansGivesIdenticalResults(int offset, int length)
        {
            var source = new byte[offset + length + offset];

            Lorem.Fill(source, 0, source.Length);

            var array = LZ4Pickler.Pickle(source, offset, length);
            var span  = LZ4Pickler.Pickle(source.AsSpan(offset, length));

            Assert.Equal(array, span);

            Assert.Equal(
                LZ4Pickler.Unpickle(array),
                LZ4Pickler.Unpickle(span.AsSpan()));
        }
示例#12
0
        public static byte[] Decompress(this byte[] data, Internal.Packets.CompressionMode compression)
        {
            switch (compression)
            {
            case Internal.Packets.CompressionMode.Deflate:
            {
                using (MemoryStream input = new MemoryStream(data))
                {
                    using (MemoryStream output = new MemoryStream())
                    {
                        using (DeflateStream dstream = new DeflateStream(input, System.IO.Compression.CompressionMode.Decompress))
                        {
                            dstream.CopyTo(output);
                        }
                        return(output.ToArray());
                    }
                }
            }

            case Internal.Packets.CompressionMode.Gzip:
            {
                using (var compressedMs = new MemoryStream(data))
                {
                    using (var decompressedMs = new MemoryStream())
                    {
                        using (var gzs = new BufferedStream(new GZipStream(compressedMs,
                                                                           System.IO.Compression.CompressionMode.Decompress), 64 * 1024))
                        {
                            gzs.CopyTo(decompressedMs);
                        }
                        return(decompressedMs.ToArray());
                    }
                }
            }

            case Internal.Packets.CompressionMode.LZ4:
                return(LZ4Pickler.Unpickle(data));

            case Internal.Packets.CompressionMode.Custom:
                return(OnCustomDecompression?.Invoke(data));

            default:
                return(data);
            }
        }
        public void PickleEntropyWithBufferWriter(
            int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
        {
            var original = new byte[length];

            new Random(seed).NextBytes(original);

            var pickledWriter   = BufferWriter.New();
            var unpickledWriter = BufferWriter.New();

            LZ4Pickler.Pickle(original, pickledWriter, level);
            var pickled = pickledWriter.WrittenSpan;

            LZ4Pickler.Unpickle(pickled, unpickledWriter);
            var unpickled = unpickledWriter.WrittenSpan;

            Tools.SameBytes(original, unpickled);
        }
        public void PickleLoremWithBufferWriter(int length, LZ4Level level = LZ4Level.L00_FAST)
        {
            var original = new byte[length];

            Lorem.Fill(original, 0, length);

            var pickledWriter   = BufferWriter.New();
            var unpickledWriter = BufferWriter.New();

            Assert.Throws <ArgumentNullException>(
                () => LZ4Pickler.Pickle(original, null, level));
            LZ4Pickler.Pickle(original, pickledWriter, level);
            var pickled = pickledWriter.WrittenSpan;

            Assert.Throws <ArgumentNullException>(
                () => LZ4Pickler.Unpickle(pickledWriter.WrittenSpan, (IBufferWriter <byte>)null));
            LZ4Pickler.Unpickle(pickled, unpickledWriter);
            var unpickled = unpickledWriter.WrittenSpan;

            Tools.SameBytes(original, unpickled);
        }
        public void Corruption()
        {
            var source = new byte[1234];

            Lorem.Fill(source, 0, source.Length);

            var array  = LZ4Pickler.Pickle(source);
            var copy   = array.AsSpan().ToArray();
            var output = source.AsSpan().ToArray();

            // pass a buffer that's too short
            Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array.AsSpan().Slice(0, 2), output));
            Assert.Throws <InvalidDataException>(() => LZ4Pickler.UnpickledSize(array.AsSpan().Slice(0, 2)));

            // corrupt the version
            array[0] = 0xff;
            Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array, output));
            Assert.Throws <InvalidDataException>(() => _ = LZ4Pickler.UnpickledSize(array));

            // corrupt the size
            array[0] = copy[0];
            array[1] = 0xff;
            Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array, output));
        }
示例#16
0
 public static byte[] UnPickle(byte[] data)
 {
     return(LZ4Pickler.Unpickle(data));
 }
示例#17
0
 public Task <byte[]> DecompressAsync(ReadOnlyMemory <byte> data)
 {
     return(Task.FromResult(LZ4Pickler.Unpickle(data.ToArray())));
 }
示例#18
0
 public byte[] Decompress(ReadOnlyMemory <byte> data)
 {
     return(LZ4Pickler.Unpickle(data.ToArray()));
 }
示例#19
0
 public byte[] Decompress(byte[] bytes)
 {
     return(LZ4Pickler.Unpickle(bytes));
 }
示例#20
0
        public ArraySegment <byte> Decompress(ReadOnlyMemory <byte> compressedData)
        {
            Guard.AgainstEmpty(compressedData, nameof(compressedData));

            return(LZ4Pickler.Unpickle(compressedData.Span));
        }
示例#21
0
 public object Decompress(byte[] data)
 {
     return(Deserialize(LZ4Pickler.Unpickle(data)));
 }
示例#22
0
 public T Unprocess <T>(byte[] bytes)
 {
     return(_processor.Unprocess <T>(LZ4Pickler.Unpickle(bytes)));
 }
示例#23
0
 public Span <byte> Decompress(Span <byte> bytes)
 {
     return(LZ4Pickler.Unpickle(bytes));
 }
示例#24
0
 public static byte[] Decompress(this byte[] input)
 {
     return(LZ4Pickler.Unpickle(input));
 }
示例#25
0
        public static void Main(string[] args)
        {
            FrontendInitializer.RegisterTypes(ServiceLocator.Default);
            FrontendInitializer.Initialize(ServiceLocator.Default);
            var vendorPresetParserService = ServiceLocator.Default.ResolveType <VendorPresetParserService>();
            var logger = new RollingInMemoryLogListener();

            LogManager.AddListener(logger);

            var pluginTestDirectory = @"C:\Program Files\VSTPlugins";
            var testResults         = new List <PluginTestResult>();

            var presetParserDictionary = vendorPresetParserService.GetPresetHandlerListByPlugin();


            var testData       = ReadTestData();
            var ignoredPlugins = ReadIgnoredPlugins();

            List <string> IgnoredPresetParsers = new List <string>();

            IgnoredPresetParsers.Add("VoidPresetParser");

            var localLogger = new MiniConsoleLogger();
            var hasIgnored  = false;

            localLogger.SetConsoleLogLevelFilter(new HashSet <LogLevel> {
                LogLevel.Error, LogLevel.Warning
            });

            if (args.Length > 0)
            {
                foreach (var key in presetParserDictionary.Keys.ToList())
                {
                    if (!presetParserDictionary[key].PresetParserType.ToLower().Contains(args[0].ToLower()))
                    {
                        presetParserDictionary.Remove(key);
                        hasIgnored = true;
                    }
                }
            }

            foreach (var presetParserKeyValue in presetParserDictionary)
            {
                var presetParser = presetParserKeyValue.Value;
                var pluginId     = presetParserKeyValue.Key;

                if (IgnoredPresetParsers.Contains(presetParser.PresetParserType))
                {
                    continue;
                }

                if (IsIgnored(ignoredPlugins, presetParser.PresetParserType, pluginId))
                {
                    continue;
                }

                Console.Write(presetParser.PresetParserType + ": ");

                var start = DateTime.Now;

                var pluginLocation = new PluginLocation
                {
                    DllPath = @"C:\Program Files\VstPlugins\Foobar.dll", IsPresent = true
                };

                var plugin = new Plugin
                {
                    VstPluginId = pluginId, PluginLocation = pluginLocation,
                    PluginInfo  = new VstPluginInfoSurrogate
                    {
                        ProgramCount = 1, Flags = VstPluginFlags.ProgramChunks, PluginID = pluginId
                    }
                };

                var stubProcess = new StubVstHostProcess();
                stubProcess.PluginId = pluginId;

                var remoteInstance = new RemotePluginInstance(stubProcess, plugin);

                presetParser.DataPersistence = new NullPresetPersistence();
                presetParser.PluginInstance  = remoteInstance;
                presetParser.RootBank        = plugin.RootBank.First();
                presetParser.Logger.Clear();
                presetParser.Logger.MirrorTo(localLogger);

                var testResult = new PluginTestResult
                {
                    VendorPresetParser = presetParser.PresetParserType,
                    PluginId           = plugin.VstPluginId
                };

                double timeForNumPresets = 0;
                double timeForDoScan     = 0;
                double totalTime         = 0;
                try
                {
                    presetParser.Init();
                    testResult.ReportedPresets = presetParser.GetNumPresets();
                    timeForNumPresets          = (DateTime.Now - start).TotalSeconds;
                    start = DateTime.Now;
                    presetParser.DoScan().GetAwaiter().GetResult();
                    timeForDoScan = (DateTime.Now - start).TotalSeconds;
                    totalTime     = timeForNumPresets + timeForDoScan;
                }
                catch (Exception e)
                {
                    testResult.Error = "Errored";
                    Console.WriteLine(e.Message);
                    Console.WriteLine(e.StackTrace);
                }

                testResult.Presets = plugin.Presets.Count;

                var timePerPreset = (totalTime / testResult.Presets) * 1000;
                // ReSharper disable once LocalizableElement
                Console.WriteLine(
                    $"{testResult.Presets} parsed in {totalTime:F3}s (avg {timePerPreset:F3}ms / Preset, DoScan {timeForDoScan:F3}s, NumPresets {timeForNumPresets:F3}s");

                var testDataEntries    = GetTestDataEntries(testData, presetParser.PresetParserType, pluginId);
                var hasTestDataEntries = testDataEntries.Count > 0;
                var testDataOk         = true;
                foreach (var preset in plugin.Presets)
                {
                    if (preset.Metadata.BankPath == "")
                    {
                        testResult.BankMissing++;
                    }

                    foreach (var testDataEntry in testDataEntries.ToList())
                    {
                        if (preset.Metadata.PresetName == testDataEntry.ProgramName &&
                            preset.Metadata.BankPath == testDataEntry.BankPath)
                        {
                            var testFilename = PathUtils.SanitizeFilename(
                                testDataEntry.PresetParser + "." + preset.OriginalMetadata.PresetName +
                                ".testdata");
                            var myDocumentsTestDataFile = Path.Combine(GetPatchFilesDirectory(), testFilename);
                            var localTestDataFile       = Path.Combine("TestData", testFilename);


                            var presetHash = testDataEntry.Hash.TrimEnd();
                            if (preset.PresetHash != presetHash)
                            {
                                var fileMessage     = "";
                                var wrongPresetData = myDocumentsTestDataFile + ".wrong";
                                testDataOk = false;

                                if (File.Exists(myDocumentsTestDataFile))
                                {
                                    fileMessage = $"Original preset data in {myDocumentsTestDataFile}" +
                                                  Environment.NewLine +
                                                  $"Current (wrong) preset data in {wrongPresetData}";
                                }
                                else
                                {
                                    fileMessage =
                                        $"Original preset data not found (expected in {myDocumentsTestDataFile})" +
                                        Environment.NewLine +
                                        $"Current (wrong) preset data in {wrongPresetData}";
                                }

                                File.WriteAllBytes(wrongPresetData, LZ4Pickler.Unpickle(
                                                       NullPresetPersistence.PresetData[preset.OriginalMetadata.SourceFile]));
                                testResult.DetailedErrors.Add(
                                    $"Found preset {testDataEntry.ProgramName} with bank path " +
                                    $"{testDataEntry.BankPath} but the preset hashes were different. " +
                                    $"Expected hash {presetHash} but found hash {preset.PresetHash}" +
                                    Environment.NewLine + Environment.NewLine + $"{fileMessage}");
                            }
                            else
                            {
                                // Check if the file exists in the output directory
                                if (!File.Exists(myDocumentsTestDataFile))
                                {
                                    if (File.Exists(localTestDataFile))
                                    {
                                        File.Copy(localTestDataFile, myDocumentsTestDataFile);
                                    }
                                    else
                                    {
                                        File.WriteAllBytes(myDocumentsTestDataFile,
                                                           LZ4Pickler.Unpickle(
                                                               NullPresetPersistence.PresetData[preset.OriginalMetadata.SourceFile]));
                                    }
                                }
                                else
                                {
                                    if (!File.Exists(localTestDataFile))
                                    {
                                        testResult.DetailedErrors.Add(
                                            $"Warning: The preset data file {testFilename} exists in the documents " +
                                            "folder but not in the source folder. Copy from documents to git folder. " +
                                            "If already done, remember to clean the presetparsertest project.");
                                    }
                                }

                                var hash = HashUtils.getIxxHash(File.ReadAllBytes(myDocumentsTestDataFile));

                                if (hash != presetHash)
                                {
                                    testResult.DetailedErrors.Add(
                                        $"Warning: The preset data file {myDocumentsTestDataFile} exists but does not match the " +
                                        $"preset hash from the reference presets. Expected: {testDataEntry.Hash} found {hash}");
                                }
                            }

                            testDataEntries.Remove(testDataEntry);
                        }
                    }
                }

                if (testDataEntries.Count > 0)
                {
                    foreach (var missingTestDataEntry in testDataEntries)
                    {
                        var presetHash = missingTestDataEntry.Hash.TrimEnd();
                        testResult.DetailedErrors.Add(
                            $"Did not find preset {missingTestDataEntry.ProgramName} with bank path " +
                            $"{missingTestDataEntry.BankPath} and hash {presetHash}");
                    }

                    testResult.IsOK = false;
                }

                if (plugin.Presets.Count > 0)
                {
                    var randomPreset = plugin.Presets.OrderBy(qu => Guid.NewGuid()).First();
                    testResult.RndHash       = randomPreset.PresetHash;
                    testResult.RndPresetName = randomPreset.Metadata.PresetName;
                    testResult.RndBankPath   = randomPreset.Metadata.BankPath;
                }

                var mockFxp = Path.Combine(Directory.GetCurrentDirectory(), "mock.fxp");
                var fxp     = new FXP();
                fxp.ReadFile(Path.Combine(Directory.GetCurrentDirectory(), "test.fxp"));
                fxp.FxID = VstUtils.PluginIdNumberToIdString(pluginId);
                fxp.WriteFile(mockFxp);
                // Test additional banks
                var bankFile = new BankFile();
                bankFile.Path     = mockFxp;
                bankFile.BankName = "Default";

                plugin.AdditionalBankFiles.Clear();
                plugin.AdditionalBankFiles.Add(bankFile);

                bool additionalBankFileCountOk = false;


                if (presetParser.GetNumPresets() == testResult.ReportedPresets + 1)
                {
                    additionalBankFileCountOk = true;
                }
                else
                {
                    testResult.Error += " additionalBankFileCount failed";
                }

                plugin.Presets.Clear();
                NullPresetPersistence.PresetData.Clear();
                presetParser.DoScan().GetAwaiter().GetResult();

                var additionalBankFileScanOk = false;

                if (plugin.Presets.Count == testResult.Presets + 1)
                {
                    additionalBankFileScanOk = true;
                }
                else
                {
                    testResult.Error += " additionalBankFileScan failed";
                }

                bool bankMissingOk = false;
                if (NumBankMissingsOk.ContainsKey(testResult.PluginId))
                {
                    if (testResult.BankMissing <= NumBankMissingsOk[testResult.PluginId])
                    {
                        bankMissingOk = true;
                    }
                }
                else
                {
                    if (testResult.BankMissing < 2)
                    {
                        bankMissingOk = true;
                    }
                }

                if (hasTestDataEntries && testDataOk && testResult.Presets > 5 && bankMissingOk &&
                    testResult.Presets == testResult.ReportedPresets && additionalBankFileCountOk &&
                    additionalBankFileScanOk)
                {
                    testResult.IsOK = true;
                }

                testResults.Add(testResult);

                NullPresetPersistence.PresetData.Clear();
            }


            var consoleTable = ConsoleTable.From(from testRes in testResults
                                                 where testRes.IsOK == false
                                                 orderby testRes.Presets
                                                 select testRes);

            Console.WriteLine(consoleTable.ToMinimalString());

            foreach (var testRes in (from testRes in testResults
                                     where testRes.DetailedErrors.Count > 0
                                     orderby testRes.Presets
                                     select testRes))
            {
                Console.WriteLine(Environment.NewLine);
                Console.WriteLine($"Detailed Errors for {testRes.VendorPresetParser}");
                Console.WriteLine($"------------------------------------------------------------");

                foreach (var detailedError in testRes.DetailedErrors)
                {
                    Console.WriteLine($"Error #{testRes.DetailedErrors.IndexOf(detailedError)}: {detailedError}");
                }
            }

            Console.WriteLine($"Stuff left: {consoleTable.Rows.Count} / {presetParserDictionary.Count}");

            foreach (var data in GlobalMethodTimeLogger.GetTopMethods())
            {
                Console.WriteLine($"{data.Name}: {data.Duration.TotalSeconds.ToString()}ms");
            }

            if (hasIgnored)
            {
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
                Console.WriteLine("Warning: Filter active!!");
            }
        }
示例#26
0
 public T Decompress <T>(byte[] data)
 {
     return(Deserialize <T>(LZ4Pickler.Unpickle(data)));
 }
示例#27
0
 public static string UnPickleString(byte[] data)
 {
     return(Encoding.Default.GetString(LZ4Pickler.Unpickle(data)));
 }
示例#28
0
 /// <summary>
 /// Decompresses the specified compressed data.
 /// </summary>
 /// <param name="compressedData">The compressed data.</param>
 /// <returns></returns>
 public byte[] Decompress(byte[] compressedData)
 {
     return(LZ4Pickler.Unpickle(compressedData));
 }