public unsafe void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
// reiterating same test, but with different overloads
fixed(byte *p = original)
pickled = LZ4Pickler.Pickle(p, original.Length, level);
fixed(byte *p = pickled)
unpickled = LZ4Pickler.Unpickle(p, pickled.Length);
Tools.SameBytes(original, unpickled);
// reiterating same test, but with offset
var copy = new byte[pickled.Length + 37];
Array.Copy(pickled, 0, copy, 37, pickled.Length);
unpickled = LZ4Pickler.Unpickle(copy, 37, pickled.Length);
Tools.SameBytes(original, unpickled);
unpickled.AsSpan().Fill(0);
LZ4Pickler.Unpickle(pickled.AsSpan(), unpickled.AsSpan());
Tools.SameBytes(original, unpickled);
}
public byte[] UnCompress(byte[] compressedProto, int offset, int length, out int outLength)
{
var unwrapped = LZ4Pickler.Unpickle(compressedProto, offset, length);
outLength = unwrapped.Length;
return(unwrapped);
}
public object FromStream(ref BytePacker packer, IDecrypter decrypter)
{
BytePacker p = packer;
if (Compression == Compression.LZ4)
{
byte[] encoded = (byte[])byteArrayConverter.Deserialize(p);
byte[] data = LZ4Pickler.Unpickle(encoded);
if (decrypter != null)
{
data = decrypter.Decrypt(data);
}
p = new BytePacker(data);
}
else if (decrypter != null)
{
try
{
byte[] data = (byte[])encArrayConverter.Deserialize(p);
data = decrypter.Decrypt(data);
p = new BytePacker(data);
}
catch (Exception e)
{
Util.Log("FromStream:" + e.Message);
}
}
return(converter.Deserialize(p));
}
/// <summary>
/// Returns the decompressed string from the bytes. This methods can only be called once.
/// </summary>
/// <returns></returns>
/// <exception cref="InvalidOperationException">Throws if this is called more than once</exception>
public string DecompressString()
{
if (_str != null)
{
return(_str);
}
lock (_locker)
{
if (_str != null)
{
// double check
return(_str);
}
if (_bytes == null)
{
throw new InvalidOperationException("Bytes have already been cleared");
}
_str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes));
_bytes = null;
}
return(_str);
}
public static byte[] Unzip(byte[] data, CompressionMethod method = CompressionMethod.Lz4)
{
if (method == CompressionMethod.Lz4)
{
return(LZ4Pickler.Unpickle(data));
}
return(null);
}
public void PickleAndUnpickle(string filename)
{
var original = File.ReadAllBytes(Tools.FindFile(filename));
var compressed = LZ4Pickler.Pickle(original);
var decompressed = LZ4Pickler.Unpickle(compressed);
Tools.SameBytes(original, decompressed);
}
public void PickleEntropy(int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
new Random(seed).NextBytes(original);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
}
public void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
}
public object FromStream(ref BytePacker packer)
{
if (Compression == Compression.LZ4)
{
byte[] encoded = (byte[])lz4converter.Deserialize(packer);
byte[] data = LZ4Pickler.Unpickle(encoded);
BytePacker lz4packer = new BytePacker(data);
return(converter.Deserialize(lz4packer));
}
else
{
return(converter.Deserialize(packer));
}
}
/// <summary>
/// Deserializing binary data to a new instance. Decompression is using LZ4 pickle self-contained format.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="objectData">Compressed data expected</param>
/// <returns></returns>
public T Unpickle <T>(Stream objectData)
{
int length = 0;
int offset = 0;
byte[] compressedBuff = null;
MemoryStream mem;
using (mem = new MemoryStream())
{
if (objectData is MemoryStream outMem)
{
#if NETCOREAPP || NETSTANDARD
if (outMem.TryGetBuffer(out var arraySegment))
{
compressedBuff = arraySegment.Array;
length = arraySegment.Count;
offset = arraySegment.Offset;
}
#else
try
{
compressedBuff = outMem.GetBuffer();
length = (int)outMem.Length;
offset = (int)outMem.Position;
}
catch (UnauthorizedAccessException)
{
// eat the error
}
#endif
}
if (compressedBuff == null)
{
objectData.CopyTo(mem);
compressedBuff = mem.GetBuffer();
length = (int)mem.Length;
offset = 0;
}
var serializedBuff = LZ4Pickler.Unpickle(compressedBuff, offset, length);
mem.Dispose();
mem = new MemoryStream(serializedBuff);
return(_serializer.Deserialize <T>(mem));
}
}
public void PicklingSpansGivesIdenticalResults(int offset, int length)
{
var source = new byte[offset + length + offset];
Lorem.Fill(source, 0, source.Length);
var array = LZ4Pickler.Pickle(source, offset, length);
var span = LZ4Pickler.Pickle(source.AsSpan(offset, length));
Assert.Equal(array, span);
Assert.Equal(
LZ4Pickler.Unpickle(array),
LZ4Pickler.Unpickle(span.AsSpan()));
}
public static byte[] Decompress(this byte[] data, Internal.Packets.CompressionMode compression)
{
switch (compression)
{
case Internal.Packets.CompressionMode.Deflate:
{
using (MemoryStream input = new MemoryStream(data))
{
using (MemoryStream output = new MemoryStream())
{
using (DeflateStream dstream = new DeflateStream(input, System.IO.Compression.CompressionMode.Decompress))
{
dstream.CopyTo(output);
}
return(output.ToArray());
}
}
}
case Internal.Packets.CompressionMode.Gzip:
{
using (var compressedMs = new MemoryStream(data))
{
using (var decompressedMs = new MemoryStream())
{
using (var gzs = new BufferedStream(new GZipStream(compressedMs,
System.IO.Compression.CompressionMode.Decompress), 64 * 1024))
{
gzs.CopyTo(decompressedMs);
}
return(decompressedMs.ToArray());
}
}
}
case Internal.Packets.CompressionMode.LZ4:
return(LZ4Pickler.Unpickle(data));
case Internal.Packets.CompressionMode.Custom:
return(OnCustomDecompression?.Invoke(data));
default:
return(data);
}
}
public void PickleEntropyWithBufferWriter(
int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
new Random(seed).NextBytes(original);
var pickledWriter = BufferWriter.New();
var unpickledWriter = BufferWriter.New();
LZ4Pickler.Pickle(original, pickledWriter, level);
var pickled = pickledWriter.WrittenSpan;
LZ4Pickler.Unpickle(pickled, unpickledWriter);
var unpickled = unpickledWriter.WrittenSpan;
Tools.SameBytes(original, unpickled);
}
public void PickleLoremWithBufferWriter(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickledWriter = BufferWriter.New();
var unpickledWriter = BufferWriter.New();
Assert.Throws <ArgumentNullException>(
() => LZ4Pickler.Pickle(original, null, level));
LZ4Pickler.Pickle(original, pickledWriter, level);
var pickled = pickledWriter.WrittenSpan;
Assert.Throws <ArgumentNullException>(
() => LZ4Pickler.Unpickle(pickledWriter.WrittenSpan, (IBufferWriter <byte>)null));
LZ4Pickler.Unpickle(pickled, unpickledWriter);
var unpickled = unpickledWriter.WrittenSpan;
Tools.SameBytes(original, unpickled);
}
public void Corruption()
{
var source = new byte[1234];
Lorem.Fill(source, 0, source.Length);
var array = LZ4Pickler.Pickle(source);
var copy = array.AsSpan().ToArray();
var output = source.AsSpan().ToArray();
// pass a buffer that's too short
Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array.AsSpan().Slice(0, 2), output));
Assert.Throws <InvalidDataException>(() => LZ4Pickler.UnpickledSize(array.AsSpan().Slice(0, 2)));
// corrupt the version
array[0] = 0xff;
Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array, output));
Assert.Throws <InvalidDataException>(() => _ = LZ4Pickler.UnpickledSize(array));
// corrupt the size
array[0] = copy[0];
array[1] = 0xff;
Assert.Throws <InvalidDataException>(() => LZ4Pickler.Unpickle(array, output));
}
public static byte[] UnPickle(byte[] data)
{
return(LZ4Pickler.Unpickle(data));
}
public Task <byte[]> DecompressAsync(ReadOnlyMemory <byte> data)
{
return(Task.FromResult(LZ4Pickler.Unpickle(data.ToArray())));
}
public byte[] Decompress(ReadOnlyMemory <byte> data)
{
return(LZ4Pickler.Unpickle(data.ToArray()));
}
public byte[] Decompress(byte[] bytes)
{
return(LZ4Pickler.Unpickle(bytes));
}
public ArraySegment <byte> Decompress(ReadOnlyMemory <byte> compressedData)
{
Guard.AgainstEmpty(compressedData, nameof(compressedData));
return(LZ4Pickler.Unpickle(compressedData.Span));
}
public object Decompress(byte[] data)
{
return(Deserialize(LZ4Pickler.Unpickle(data)));
}
public T Unprocess <T>(byte[] bytes)
{
return(_processor.Unprocess <T>(LZ4Pickler.Unpickle(bytes)));
}
public Span <byte> Decompress(Span <byte> bytes)
{
return(LZ4Pickler.Unpickle(bytes));
}
public static byte[] Decompress(this byte[] input)
{
return(LZ4Pickler.Unpickle(input));
}
public static void Main(string[] args)
{
FrontendInitializer.RegisterTypes(ServiceLocator.Default);
FrontendInitializer.Initialize(ServiceLocator.Default);
var vendorPresetParserService = ServiceLocator.Default.ResolveType <VendorPresetParserService>();
var logger = new RollingInMemoryLogListener();
LogManager.AddListener(logger);
var pluginTestDirectory = @"C:\Program Files\VSTPlugins";
var testResults = new List <PluginTestResult>();
var presetParserDictionary = vendorPresetParserService.GetPresetHandlerListByPlugin();
var testData = ReadTestData();
var ignoredPlugins = ReadIgnoredPlugins();
List <string> IgnoredPresetParsers = new List <string>();
IgnoredPresetParsers.Add("VoidPresetParser");
var localLogger = new MiniConsoleLogger();
var hasIgnored = false;
localLogger.SetConsoleLogLevelFilter(new HashSet <LogLevel> {
LogLevel.Error, LogLevel.Warning
});
if (args.Length > 0)
{
foreach (var key in presetParserDictionary.Keys.ToList())
{
if (!presetParserDictionary[key].PresetParserType.ToLower().Contains(args[0].ToLower()))
{
presetParserDictionary.Remove(key);
hasIgnored = true;
}
}
}
foreach (var presetParserKeyValue in presetParserDictionary)
{
var presetParser = presetParserKeyValue.Value;
var pluginId = presetParserKeyValue.Key;
if (IgnoredPresetParsers.Contains(presetParser.PresetParserType))
{
continue;
}
if (IsIgnored(ignoredPlugins, presetParser.PresetParserType, pluginId))
{
continue;
}
Console.Write(presetParser.PresetParserType + ": ");
var start = DateTime.Now;
var pluginLocation = new PluginLocation
{
DllPath = @"C:\Program Files\VstPlugins\Foobar.dll", IsPresent = true
};
var plugin = new Plugin
{
VstPluginId = pluginId, PluginLocation = pluginLocation,
PluginInfo = new VstPluginInfoSurrogate
{
ProgramCount = 1, Flags = VstPluginFlags.ProgramChunks, PluginID = pluginId
}
};
var stubProcess = new StubVstHostProcess();
stubProcess.PluginId = pluginId;
var remoteInstance = new RemotePluginInstance(stubProcess, plugin);
presetParser.DataPersistence = new NullPresetPersistence();
presetParser.PluginInstance = remoteInstance;
presetParser.RootBank = plugin.RootBank.First();
presetParser.Logger.Clear();
presetParser.Logger.MirrorTo(localLogger);
var testResult = new PluginTestResult
{
VendorPresetParser = presetParser.PresetParserType,
PluginId = plugin.VstPluginId
};
double timeForNumPresets = 0;
double timeForDoScan = 0;
double totalTime = 0;
try
{
presetParser.Init();
testResult.ReportedPresets = presetParser.GetNumPresets();
timeForNumPresets = (DateTime.Now - start).TotalSeconds;
start = DateTime.Now;
presetParser.DoScan().GetAwaiter().GetResult();
timeForDoScan = (DateTime.Now - start).TotalSeconds;
totalTime = timeForNumPresets + timeForDoScan;
}
catch (Exception e)
{
testResult.Error = "Errored";
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
testResult.Presets = plugin.Presets.Count;
var timePerPreset = (totalTime / testResult.Presets) * 1000;
// ReSharper disable once LocalizableElement
Console.WriteLine(
$"{testResult.Presets} parsed in {totalTime:F3}s (avg {timePerPreset:F3}ms / Preset, DoScan {timeForDoScan:F3}s, NumPresets {timeForNumPresets:F3}s");
var testDataEntries = GetTestDataEntries(testData, presetParser.PresetParserType, pluginId);
var hasTestDataEntries = testDataEntries.Count > 0;
var testDataOk = true;
foreach (var preset in plugin.Presets)
{
if (preset.Metadata.BankPath == "")
{
testResult.BankMissing++;
}
foreach (var testDataEntry in testDataEntries.ToList())
{
if (preset.Metadata.PresetName == testDataEntry.ProgramName &&
preset.Metadata.BankPath == testDataEntry.BankPath)
{
var testFilename = PathUtils.SanitizeFilename(
testDataEntry.PresetParser + "." + preset.OriginalMetadata.PresetName +
".testdata");
var myDocumentsTestDataFile = Path.Combine(GetPatchFilesDirectory(), testFilename);
var localTestDataFile = Path.Combine("TestData", testFilename);
var presetHash = testDataEntry.Hash.TrimEnd();
if (preset.PresetHash != presetHash)
{
var fileMessage = "";
var wrongPresetData = myDocumentsTestDataFile + ".wrong";
testDataOk = false;
if (File.Exists(myDocumentsTestDataFile))
{
fileMessage = $"Original preset data in {myDocumentsTestDataFile}" +
Environment.NewLine +
$"Current (wrong) preset data in {wrongPresetData}";
}
else
{
fileMessage =
$"Original preset data not found (expected in {myDocumentsTestDataFile})" +
Environment.NewLine +
$"Current (wrong) preset data in {wrongPresetData}";
}
File.WriteAllBytes(wrongPresetData, LZ4Pickler.Unpickle(
NullPresetPersistence.PresetData[preset.OriginalMetadata.SourceFile]));
testResult.DetailedErrors.Add(
$"Found preset {testDataEntry.ProgramName} with bank path " +
$"{testDataEntry.BankPath} but the preset hashes were different. " +
$"Expected hash {presetHash} but found hash {preset.PresetHash}" +
Environment.NewLine + Environment.NewLine + $"{fileMessage}");
}
else
{
// Check if the file exists in the output directory
if (!File.Exists(myDocumentsTestDataFile))
{
if (File.Exists(localTestDataFile))
{
File.Copy(localTestDataFile, myDocumentsTestDataFile);
}
else
{
File.WriteAllBytes(myDocumentsTestDataFile,
LZ4Pickler.Unpickle(
NullPresetPersistence.PresetData[preset.OriginalMetadata.SourceFile]));
}
}
else
{
if (!File.Exists(localTestDataFile))
{
testResult.DetailedErrors.Add(
$"Warning: The preset data file {testFilename} exists in the documents " +
"folder but not in the source folder. Copy from documents to git folder. " +
"If already done, remember to clean the presetparsertest project.");
}
}
var hash = HashUtils.getIxxHash(File.ReadAllBytes(myDocumentsTestDataFile));
if (hash != presetHash)
{
testResult.DetailedErrors.Add(
$"Warning: The preset data file {myDocumentsTestDataFile} exists but does not match the " +
$"preset hash from the reference presets. Expected: {testDataEntry.Hash} found {hash}");
}
}
testDataEntries.Remove(testDataEntry);
}
}
}
if (testDataEntries.Count > 0)
{
foreach (var missingTestDataEntry in testDataEntries)
{
var presetHash = missingTestDataEntry.Hash.TrimEnd();
testResult.DetailedErrors.Add(
$"Did not find preset {missingTestDataEntry.ProgramName} with bank path " +
$"{missingTestDataEntry.BankPath} and hash {presetHash}");
}
testResult.IsOK = false;
}
if (plugin.Presets.Count > 0)
{
var randomPreset = plugin.Presets.OrderBy(qu => Guid.NewGuid()).First();
testResult.RndHash = randomPreset.PresetHash;
testResult.RndPresetName = randomPreset.Metadata.PresetName;
testResult.RndBankPath = randomPreset.Metadata.BankPath;
}
var mockFxp = Path.Combine(Directory.GetCurrentDirectory(), "mock.fxp");
var fxp = new FXP();
fxp.ReadFile(Path.Combine(Directory.GetCurrentDirectory(), "test.fxp"));
fxp.FxID = VstUtils.PluginIdNumberToIdString(pluginId);
fxp.WriteFile(mockFxp);
// Test additional banks
var bankFile = new BankFile();
bankFile.Path = mockFxp;
bankFile.BankName = "Default";
plugin.AdditionalBankFiles.Clear();
plugin.AdditionalBankFiles.Add(bankFile);
bool additionalBankFileCountOk = false;
if (presetParser.GetNumPresets() == testResult.ReportedPresets + 1)
{
additionalBankFileCountOk = true;
}
else
{
testResult.Error += " additionalBankFileCount failed";
}
plugin.Presets.Clear();
NullPresetPersistence.PresetData.Clear();
presetParser.DoScan().GetAwaiter().GetResult();
var additionalBankFileScanOk = false;
if (plugin.Presets.Count == testResult.Presets + 1)
{
additionalBankFileScanOk = true;
}
else
{
testResult.Error += " additionalBankFileScan failed";
}
bool bankMissingOk = false;
if (NumBankMissingsOk.ContainsKey(testResult.PluginId))
{
if (testResult.BankMissing <= NumBankMissingsOk[testResult.PluginId])
{
bankMissingOk = true;
}
}
else
{
if (testResult.BankMissing < 2)
{
bankMissingOk = true;
}
}
if (hasTestDataEntries && testDataOk && testResult.Presets > 5 && bankMissingOk &&
testResult.Presets == testResult.ReportedPresets && additionalBankFileCountOk &&
additionalBankFileScanOk)
{
testResult.IsOK = true;
}
testResults.Add(testResult);
NullPresetPersistence.PresetData.Clear();
}
var consoleTable = ConsoleTable.From(from testRes in testResults
where testRes.IsOK == false
orderby testRes.Presets
select testRes);
Console.WriteLine(consoleTable.ToMinimalString());
foreach (var testRes in (from testRes in testResults
where testRes.DetailedErrors.Count > 0
orderby testRes.Presets
select testRes))
{
Console.WriteLine(Environment.NewLine);
Console.WriteLine($"Detailed Errors for {testRes.VendorPresetParser}");
Console.WriteLine($"------------------------------------------------------------");
foreach (var detailedError in testRes.DetailedErrors)
{
Console.WriteLine($"Error #{testRes.DetailedErrors.IndexOf(detailedError)}: {detailedError}");
}
}
Console.WriteLine($"Stuff left: {consoleTable.Rows.Count} / {presetParserDictionary.Count}");
foreach (var data in GlobalMethodTimeLogger.GetTopMethods())
{
Console.WriteLine($"{data.Name}: {data.Duration.TotalSeconds.ToString()}ms");
}
if (hasIgnored)
{
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
Console.WriteLine("Warning: Filter active!!");
}
}
public T Decompress <T>(byte[] data)
{
return(Deserialize <T>(LZ4Pickler.Unpickle(data)));
}
public static string UnPickleString(byte[] data)
{
return(Encoding.Default.GetString(LZ4Pickler.Unpickle(data)));
}
/// <summary>
/// Decompresses the specified compressed data.
/// </summary>
/// <param name="compressedData">The compressed data.</param>
/// <returns></returns>
public byte[] Decompress(byte[] compressedData)
{
return(LZ4Pickler.Unpickle(compressedData));
}