/// <summary>
/// 载入站点配置
/// </summary>
/// <param name="siteShortName">站点短名</param>
public static void LoadSiteConfig(string siteShortName)
{
if (string.IsNullOrWhiteSpace(siteShortName))
{
return;
}
string content = string.Empty, iv = content, siteini = $"{SitePacksPath}{siteShortName}.ini";
if (string.IsNullOrWhiteSpace(siteShortName) || inis.Any(d => d.Key == siteShortName))
{
return;
}
if (File.Exists(siteini))
{
//读入并解压
content = File.ReadAllText(siteini);
if (string.IsNullOrWhiteSpace(content) || content.Length < 7)
{
return;
}
content = Encoding.UTF8.GetString(LZ4Codec.Unwrap(Convert.FromBase64String(content)));
iv = content.Substring(0, 6);
content = AESHelper.AesDecrypt(content.Substring(6), aesKey, iv);
//解析并入库
IniData iniData = iniParser.Parser.Parse(content);
inis.Add(siteShortName, iniData);
return;
}
inis.Add(siteShortName, new IniData());
}
/// <inheritdoc/>
protected override byte[] BaseDecompress(byte[] compressedBytes)
{
var target = new byte[compressedBytes.Length * 255].AsSpan();
int decoded = LZ4Codec.Decode(compressedBytes, target);
return(target.Slice(0, decoded).ToArray());
}
public byte[] UncompressedData()
{
using (MemoryMappedFile file = MemoryMappedFile.CreateFromFile(Bundle.FileName, FileMode.Open))
{
using (MemoryMappedViewStream viewstream = file.CreateViewStream(PageOFfset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case "None":
{
using (MemoryStream destination = new MemoryStream())
{
viewstream.CopyTo(destination);
return(destination.ToArray());
}
}
case "Lz4":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
return(uncompressed);
}
case "Snappy":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
return(uncompressed);
}
case "Doboz":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
return(uncompressed);
}
case "Zlib":
{
using (MemoryStream destination = new MemoryStream())
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(destination);
return(destination.ToArray());
}
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
}
}
}
public IList <CloudPoint> deserialze(string data, int height, int width)
{
byte[] vs = LZ4Codec.Unwrap(Convert.FromBase64String(data));
IList <CloudPoint> points = new List <CloudPoint>(height * width);
for (int i = 0; i < width * height; i++)
{
if (vs[i] != 0x00)
{
// Convert R3G3B2 to R8G8B8
byte color = vs[i];
byte mask = 0xE0;
float r = color & mask;
float g = (color << 3) & mask;
float b = (color << 6) & mask;
float x = i % width;
float y = i / width;
float z = bytesToShort(vs[i * 2 + width * height + 1], vs[i * 2 + width * height]);
points.Add(new CloudPoint(r / 256, g / 256, b / 256, x, y, z));
}
}
return(points);
}
public void CanEncodePartOfBufferWithSpan(int offset, int sourceLength)
{
var sourceTotal = offset + sourceLength + offset;
var targetLength = LZ4Codec.MaximumOutputSize(sourceTotal);
var targetTotal = offset + targetLength + offset;
var source = new byte[sourceTotal];
var encoded = new byte[targetTotal];
var decoded = new byte[sourceTotal];
Lorem.Fill(source, 0, source.Length);
Fill(encoded, 0xCD);
Fill(decoded, 0xCD);
var encodedLength = LZ4Codec.Encode(
source.AsSpan(offset, sourceLength),
encoded.AsSpan(offset, targetLength));
Check(encoded, 0, offset, 0xCD);
Check(encoded, offset + encodedLength, offset, 0xCD);
var decodedLength = LZ4Codec.Decode(
encoded.AsSpan(offset, encodedLength),
decoded.AsSpan(offset, sourceLength));
Assert.Equal(sourceLength, decodedLength);
Check(decoded, 0, offset, 0xCD);
Check(decoded, offset + decodedLength, offset, 0xCD);
Check(decoded, offset, decodedLength, source);
}
private static string ConvertSerializedShader(Shader shader)
{
var length = shader.platforms.Length;
var shaderPrograms = new ShaderProgram[length];
for (var i = 0; i < length; i++)
{
for (var j = 0; j < shader.offsets[i].Length; j++)
{
var offset = shader.offsets[i][j];
var compressedLength = shader.compressedLengths[i][j];
var decompressedLength = shader.decompressedLengths[i][j];
var decompressedBytes = new byte[decompressedLength];
LZ4Codec.Decode(shader.compressedBlob, (int)offset, (int)compressedLength, decompressedBytes, 0, (int)decompressedLength);
using (var blobReader = new BinaryReader(new MemoryStream(decompressedBytes)))
{
if (j == 0)
{
shaderPrograms[i] = new ShaderProgram(blobReader, shader.version);
}
shaderPrograms[i].Read(blobReader, j);
}
}
}
return(ConvertSerializedShader(shader.m_ParsedForm, shader.platforms, shaderPrograms));
}
private void ReadFileListV15(BinaryReader reader, Package package)
{
int numFiles = reader.ReadInt32();
int compressedSize = reader.ReadInt32();
byte[] compressedFileList = reader.ReadBytes(compressedSize);
int fileBufferSize = Marshal.SizeOf(typeof(FileEntry15)) * numFiles;
var uncompressedList = new byte[fileBufferSize];
int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
if (uncompressedSize != fileBufferSize)
{
string msg = $"LZ4 compressor disagrees about the size of file headers; expected {fileBufferSize}, got {uncompressedSize}";
throw new InvalidDataException(msg);
}
var ms = new MemoryStream(uncompressedList);
var msr = new BinaryReader(ms);
var entries = new FileEntry15[numFiles];
BinUtils.ReadStructs(msr, entries);
foreach (var entry in entries)
{
package.Files.Add(PackagedFileInfo.CreateFromEntry(entry, _streams[entry.ArchivePart]));
}
}
/// <summary>
/// Called ona reply to a historical data request
/// </summary>
private void HandleHistoricalDataRequestReply()
{
using (var ms = new MemoryStream())
{
// 2nd message part: the HistoricalDataRequest object that was used to make the request
bool hasMore;
var requestBuffer = _historicalDataSocket.ReceiveFrameBytes(out hasMore);
if (!hasMore)
{
return;
}
var request = MyUtils.ProtoBufDeserialize <HistoricalDataRequest>(requestBuffer, ms);
// 3rd message part: the size of the uncompressed, serialized data. Necessary for decompression.
var sizeBuffer = _historicalDataSocket.ReceiveFrameBytes(out hasMore);
if (!hasMore)
{
return;
}
var outputSize = BitConverter.ToInt32(sizeBuffer, 0);
// 4th message part: the compressed serialized data.
var dataBuffer = _historicalDataSocket.ReceiveFrameBytes();
var decompressed = LZ4Codec.Decode(dataBuffer, 0, dataBuffer.Length, outputSize);
var data = MyUtils.ProtoBufDeserialize <List <OHLCBar> >(decompressed, ms);
// Remove from pending requests
lock (_pendingHistoricalRequestsLock)
{
PendingHistoricalRequests.RemoveAll(x => x.RequestID == request.RequestID);
}
RaiseEvent(HistoricalDataReceived, this, new HistoricalDataEventArgs(request, data));
}
}
public static async Task DumpAndUploadPlayLog()
{
var count = 10000;
SushiDebug.Log($"Dumping log for last {count:n0} log entries...");
var dummyLogEntryBytes = GetLogEntryBytes(BalloonLogEntry.Type.DummyLogRecord, 0, 0);
using (var readLogStream = instance.OpenReadLogStream())
{
readLogStream.Seek(Math.Max(0, readLogStream.Length - count * dummyLogEntryBytes.Length), SeekOrigin.Begin);
var bytes = new byte[count * dummyLogEntryBytes.Length];
var readByteCount = readLogStream.Read(bytes, 0, bytes.Length);
SushiDebug.Log(
$"{readByteCount:n0} bytes ({readByteCount / dummyLogEntryBytes.Length:n0} log entries) read.");
if (readByteCount % dummyLogEntryBytes.Length != 0)
{
Debug.LogError(
$"Log dump failed! readByteCount={readByteCount:n0}, logEntrySize={dummyLogEntryBytes.Length:n0}. Abort!");
return;
}
var maxOutBytesLength = LZ4Codec.MaximumOutputLength(readByteCount);
SushiDebug.Log($"Maximum compressed log: {maxOutBytesLength:n0} bytes");
var outBytes = new byte[maxOutBytesLength];
var outBytesLength = LZ4Codec.Encode(bytes, 0, readByteCount, outBytes, 0, outBytes.Length);
SushiDebug.Log($"Compressed log size: {outBytesLength:n0} bytes");
outBytes = outBytes.Take(outBytesLength).ToArray();
await instance.UploadPlayLogAsync(outBytes, readByteCount);
}
}
public static int SerializeToBlock <T>(ref byte[] bytes, int offset, T obj, IFormatterResolver resolver)
{
var serializedData = MessagePackSerializer.SerializeUnsafe(obj, resolver);
if (serializedData.Count < NotCompressionSize)
{
// can't write direct, shoganai...
MessagePackBinary.EnsureCapacity(ref bytes, offset, serializedData.Count);
Buffer.BlockCopy(serializedData.Array, serializedData.Offset, bytes, offset, serializedData.Count);
return(serializedData.Count);
}
else
{
var maxOutCount = LZ4Codec.MaximumOutputLength(serializedData.Count);
MessagePackBinary.EnsureCapacity(ref bytes, offset, 6 + 5 + maxOutCount); // (ext header size + fixed length size)
// acquire ext header position
var extHeaderOffset = offset;
offset += (6 + 5);
// write body
var lz4Length = LZ4Codec.Encode(serializedData.Array, serializedData.Offset, serializedData.Count, bytes, offset, bytes.Length - offset);
// write extension header(always 6 bytes)
extHeaderOffset += MessagePackBinary.WriteExtensionFormatHeaderForceExt32Block(ref bytes, extHeaderOffset, (sbyte)ExtensionTypeCode, lz4Length + 5);
// write length(always 5 bytes)
MessagePackBinary.WriteInt32ForceInt32Block(ref bytes, extHeaderOffset, serializedData.Count);
return(6 + 5 + lz4Length);
}
}
public override void Flush()
{
var maxCompressedSize = LZ4Codec.MaximumOutputLength(_uncompressedBuffer.Length);
if (_compressedBuffer == null || _compressedBuffer.Length < maxCompressedSize)
{
_compressedBuffer = new byte[maxCompressedSize];
}
var compressedSize = LZ4Codec.Encode(_uncompressedBuffer, 0, _bufferLen, _compressedBuffer, 0, _compressedBuffer.Length);
if (compressedSize >= _bufferLen)
{
// Lz4 allows to write non-compressed block.
// Reuse _ncompressedBuffer which is not needed anymore to serialize block size
LittleEndianConverter.Write((uint)(_bufferLen | 1 << 31), _compressedBuffer, 0); // highest bit set indicates no compression
_base.Write(_compressedBuffer, 0, 4);
_base.Write(_uncompressedBuffer, 0, _bufferLen);
}
else
{
LittleEndianConverter.Write((uint)compressedSize, _uncompressedBuffer, 0);
_base.Write(_uncompressedBuffer, 0, 4);
_base.Write(_compressedBuffer, 0, compressedSize);
}
_bufferLen = 0;
}
static ArraySegment <byte> ToLZ4BinaryCore(ArraySegment <byte> serializedData)
{
if (serializedData.Count < NotCompressionSize)
{
return(serializedData);
}
else
{
var offset = 0;
var buffer = LZ4MemoryPool.GetBuffer();
var maxOutCount = LZ4Codec.MaximumOutputLength(serializedData.Count);
if (buffer.Length + 6 + 5 < maxOutCount) // (ext header size + fixed length size)
{
buffer = new byte[6 + 5 + maxOutCount];
}
// acquire ext header position
var extHeaderOffset = offset;
offset += (6 + 5);
// write body
var lz4Length = LZ4Codec.Encode(serializedData.Array, serializedData.Offset, serializedData.Count, buffer, offset, buffer.Length - offset);
// write extension header(always 6 bytes)
extHeaderOffset += MessagePackBinary.WriteExtensionFormatHeaderForceExt32Block(ref buffer, extHeaderOffset, (sbyte)ExtensionTypeCode, lz4Length + 5);
// write length(always 5 bytes)
MessagePackBinary.WriteInt32ForceInt32Block(ref buffer, extHeaderOffset, serializedData.Count);
return(new ArraySegment <byte>(buffer, 0, 6 + 5 + lz4Length));
}
}
public static void Decompress(byte[] compressed, int compressedOffset, int compressedSize, byte[] uncompressed, int uncompressedOffset, int uncompressedSize, CompressionMethod method, FArchive?reader = null)
{
using var srcStream = new MemoryStream(compressed, compressedOffset, compressedSize, false)
{
Position = 0
};
switch (method)
{
case CompressionMethod.None:
Buffer.BlockCopy(compressed, compressedOffset, uncompressed, uncompressedOffset, compressedSize);
return;
case CompressionMethod.Zlib:
var zlib = new ZlibStream(srcStream, CompressionMode.Decompress);
zlib.Read(uncompressed, uncompressedOffset, uncompressedSize);
zlib.Dispose();
return;
case CompressionMethod.Gzip:
var gzip = new GZipStream(srcStream, CompressionMode.Decompress);
gzip.Read(uncompressed, uncompressedOffset, uncompressedSize);
gzip.Dispose();
return;
case CompressionMethod.Oodle:
Oodle.Decompress(compressed, compressedOffset, compressedSize, uncompressed, uncompressedOffset, uncompressedSize, reader);
return;
case CompressionMethod.LZ4:
var uncompressedBuffer = new byte[uncompressedSize + uncompressedSize / 255 + 16]; // LZ4_compressBound(uncompressedSize)
int result;
#if USE_LZ4_NATIVE_LIB
unsafe
{
fixed(byte *compressedPtr = compressed, uncompressedBufferPtr = uncompressedBuffer)
{
result = LZ4.LZ4_decompress_safe(compressedPtr + compressedOffset, uncompressedBufferPtr, compressedSize, uncompressedBuffer.Length);
}
}
#else
result = LZ4Codec.Decode(compressed, compressedOffset, compressedSize, uncompressedBuffer, 0, uncompressedBuffer.Length);
#endif
Buffer.BlockCopy(uncompressedBuffer, 0, uncompressed, uncompressedOffset, uncompressedSize);
if (result != uncompressedSize)
{
throw new FileLoadException($"Failed to decompress LZ4 data (Expected: {uncompressedSize}, Result: {result})");
}
return;
default:
if (reader != null)
{
throw new UnknownCompressionMethodException(reader, $"Compression method \"{method}\" is unknown");
}
else
{
throw new UnknownCompressionMethodException($"Compression method \"{method}\" is unknown");
}
}
}
public void TestEncodeDecodeOnFileFragments(string fileName, int offsetEncode, int offsetDecode)
{
var rand = new Random();
using (var file = File.Open(fileName, FileMode.Open, FileAccess.Read))
{
for (var i = 0; i < 100; i++)
{
var maxSize = file.Length;
var offsetLength = rand.Next((int)maxSize * 3 / 4);
file.Seek(offsetLength, SeekOrigin.Begin);
var original = new byte[rand.Next(100, (int)maxSize - offsetLength)];
file.Read(original, 0, original.Length);
file.Seek(0, SeekOrigin.Begin);
var lengthForEncodeInput = original.Length - offsetEncode;
var encoded = LZ4Codec.Encode(original, offsetEncode, lengthForEncodeInput);
if (offsetDecode > 0)
{
encoded = EnlargeArray(encoded, offsetDecode);
}
var decoded = LZ4Codec.Decode(encoded, offsetDecode, encoded.Length - offsetDecode,
lengthForEncodeInput);
Assert.Throws <ArgumentException>(() =>
LZ4Codec.Decode(encoded, offsetDecode, encoded.Length - offsetDecode,
lengthForEncodeInput - 1));
decoded.SequenceEqual(original.Skip(offsetEncode)).Should().BeTrue();
}
}
}
protected override byte[] Decompress(Stream compressed, out UInt128 compressedHash)
{
var header = new byte[9];
int read = 0;
do
{
read += compressed.Read(header, read, header.Length - read);
} while (read < header.Length);
if (header[0] != Header[0])
{
throw new FormatException($"Invalid header value {header[0]}");
}
var compressedSize = BitConverter.ToInt32(header, 1);
var uncompressedSize = BitConverter.ToInt32(header, 5);
read = 0;
compressedSize -= header.Length;
var compressedBytes = new byte[compressedSize + header.Length];
Array.Copy(header, 0, compressedBytes, 0, header.Length);
do
{
read += compressed.Read(compressedBytes, header.Length + read, compressedSize - read);
} while (read < compressedSize);
compressedHash = ClickHouseCityHash.CityHash128(compressedBytes);
return(LZ4Codec.Decode(compressedBytes, header.Length, compressedSize, uncompressedSize));
}
public void AddChatNode(string str)
{
ChatInfoNodeData data = null;
try
{
var bytes = Convert.FromBase64String(str);
var unwrapped = LZ4Codec.Decode32(bytes, 0, bytes.Length, decodeBuffer, 0, decodeBuffer.Length, false);
using (var ms = new MemoryStream(decodeBuffer, 0, unwrapped, false))
{
data = Serializer.Deserialize <ChatInfoNodeData>(ms);
}
}
catch (Exception)
{
// ignored
}
if (data == null)
{
mCharInfoNodes.Add(new CharInfoNode(ShouldCheckSensitiveWord(), str));
return;
}
if (data.Type == (int)eChatLinkType.Dictionary)
{
AnalyseDictionaryNode(data);
return;
}
var infoNode = new CharInfoNode(ShouldCheckSensitiveWord(), "", data);
mCharInfoNodes.Add(infoNode);
}
public void Load(PrimitiveReader reader)
{
// checking magic
var magic = reader.ReadUInt32();
if (magic != Magic)
{
throw new InvalidOperationException("Memory: Cannot resume state from stream: Invalid magic.");
}
SegmentSize = reader.ReadInt32();
size = reader.ReadUInt32();
if (emptyCtorUsed)
{
Init();
}
var realSegmentsCount = 0;
for (var i = 0; i < segments.Length; i++)
{
var isTouched = reader.ReadBoolean();
if (!isTouched)
{
continue;
}
var compressedSegmentSize = reader.ReadInt32();
var compressedBuffer = reader.ReadBytes(compressedSegmentSize);
TouchSegment(i);
realSegmentsCount++;
LZ4Codec.Decode64(compressedBuffer, segments[i], SegmentSize);
}
this.NoisyLog(string.Format("{0} segments loaded from stream, of which {1} had content.", segments.Length, realSegmentsCount));
}
protected virtual byte[] CompressContent(Request request)
{
var encoding = string.IsNullOrEmpty(request.EncodingName) ? Encoding.UTF8 : Encoding.GetEncoding(request.EncodingName);
var bytes = encoding.GetBytes(request.Content);
switch (request.CompressMode)
{
case CompressMode.Lz4:
{
bytes = LZ4Codec.Wrap(bytes);
break;
}
case CompressMode.None:
{
break;
}
default:
{
throw new NotImplementedException(request.CompressMode.ToString());
}
}
return(bytes);
}
/// <summary>
/// Extract existing memory-mapped-file,
/// decompress with the proper algorithm.
/// </summary>
/// <param name="output"></param>
public void ExtractExistingMMF(Stream output, MemoryMappedFile memorymappedbundle)
{
/* var hash = *//*@"Global\" + *//*Archive.FileName.GetHashMD5();
* System.Console.WriteLine(hash);
* using (MemoryMappedFile mmf = MemoryMappedFile.OpenExisting(hash, MemoryMappedFileRights.Read, HandleInheritability.Inheritable))
*/
using (var viewstream = memorymappedbundle.CreateViewStream(PageOffset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case "None":
{
viewstream.CopyTo(output);
break;
}
case "Lz4":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Snappy":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Doboz":
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case "Zlib":
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(output);
break;
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
viewstream.Close();
}
}
public static byte[] ToBytes(this object message)
{
message.NotNull(nameof(message));
var bytes = MessagePackSerializer.Typeless.Serialize(message);
return(LZ4Codec.Wrap(bytes));
}
public UserGet Decompress(byte[] inFile)
{
using (var src = new MemoryStream(LZ4Codec.Unwrap(inFile)))
{
src.Position = 0;
return(Serializer.Deserialize <UserGet>(src));
}
//using (var input = new MemoryStream(inFile))
//using (var output = new MemoryStream())
//{
// var decoder = new SevenZip.SDK.Compress.LZMA.Decoder();
// byte[] properties = new byte[5];
// if (input.Read(properties, 0, 5) != 5)
// throw (new Exception("input .lzma is too short"));
// decoder.SetDecoderProperties(properties);
// long outSize = 0;
// for (int i = 0; i < 8; i++)
// {
// int v = input.ReadByte();
// if (v < 0)
// throw (new Exception("Can't Read 1"));
// outSize |= ((long)(byte)v) << (8 * i);
// }
// long compressedSize = input.Length - input.Position;
// decoder.Code(input, output, compressedSize, outSize, null);
// output.Position = 0;
// return ProtoBuf.Serializer.Deserialize<UserGet>(output);
//}
}
public static async Task <object> DeserializeAsync(this byte[] bytes, CancellationToken cancellationToken)
{
bytes = LZ4Codec.Unwrap(bytes);
var stream = new MemoryStream(bytes);
return(await MessagePackSerializer.Typeless.DeserializeAsync(stream, null, cancellationToken));
}
protected override Stream Decompress(Stream uncompressed)
{
var header = new byte[9];
int read = 0;
do
{
read += uncompressed.Read(header, read, header.Length - read);
} while (read < header.Length);
if (header[0] != Header[0])
{
throw new FormatException($"Invalid header value {header[0]}");
}
var compressedSize = BitConverter.ToInt32(header, 1);
var uncompressedSize = BitConverter.ToInt32(header, 5);
read = 0;
compressedSize -= header.Length;
var cdata = new byte[compressedSize];
do
{
read += uncompressed.Read(cdata, read, compressedSize - read);
} while (read < compressedSize);
return(new MemoryStream(LZ4Codec.Decode(cdata, 0, compressedSize, uncompressedSize)));
}
/// <summary>
/// Populates StateBytes and StateBytesCompressed
/// using the current GameState object
/// </summary>
public void TakeSnapshot()
{
StateBytes = GetStateBytes();
StateBytesCompressed = LZ4Codec.Wrap(StateBytes);
Console.WriteLine(string.Format("Game state length = {0} ({1} compressed)", StateBytes.Length, StateBytesCompressed.Length));
}
/// <inheritdoc/>
protected override byte[] BaseCompress(byte[] bytes)
{
var target = new byte[LZ4Codec.MaximumOutputSize(bytes.Length)].AsSpan();
int compressedBytesSize = LZ4Codec.Encode(bytes, target, Level);
return(target.Slice(0, compressedBytesSize).ToArray());
}
private static Byte[] DecompressContent(CompressType compressType, Byte[] orignData, Int32?rawSize)
{
switch (compressType)
{
case CompressType.None:
{
return(orignData);
}
case CompressType.Lz4:
{
if (!rawSize.HasValue)
{
throw new ArgumentException($"{LogHeaders.BodyRawSize} is required when using [lz4] compress.");
}
var rawData = LZ4Codec.Decode(orignData, 0, orignData.Length, rawSize.Value);
return(rawData);
}
case CompressType.Deflate:
{
var rawData = ZlibStream.UncompressBuffer(orignData);
return(rawData);
}
default:
{
throw new ArgumentOutOfRangeException(nameof(compressType), compressType, null);
}
}
}
/// <summary>
/// Loads Package Index File
/// </summary>
/// <param name="filePath">Path to Package file</param>
public bool Load(string filePath)
{
using (BinaryReader reader = new BinaryReader(File.OpenRead(filePath)))
{
// Check Magic
if (reader.ReadInt32() != 0x20494E57)
{
return(false);
}
else
{
// Version / Count
int version = reader.ReadInt16();
int count = reader.ReadInt32();
// Buffer Sizes
int compressedSize = reader.ReadInt32();
int decompressedSize = reader.ReadInt32();
// Decode the Buffer
byte[] buffer = LZ4Codec.Decode(reader.ReadBytes(compressedSize), 0, compressedSize, decompressedSize);
// Read the package
using (BinaryReader internalReader = new BinaryReader(new MemoryStream(buffer)))
// Loop Count
for (int i = 0; i < count; i++)
{
// Read ID and String
Entries[internalReader.ReadUInt64() & 0xFFFFFFFFFFFFFFF] = ReadNullTerminatedString(internalReader);
}
}
}
return(true);
}
protected virtual byte[] CompressContent(Request request)
{
if (string.IsNullOrWhiteSpace(request.Body))
{
return(new byte[0]);
}
var encoding = string.IsNullOrEmpty(request.Encoding)
? Encoding.UTF8
: Encoding.GetEncoding(request.Encoding);
var bytes = encoding.GetBytes(request.Body);
switch (request.Compression)
{
case Compression.Lz4:
{
bytes = LZ4Codec.Wrap(bytes);
break;
}
case Compression.None:
{
break;
}
default:
{
throw new NotImplementedException(request.Compression.ToString());
}
}
return(bytes);
}
public async Task SerializeAsync(Stream stream, object message, Type type, CancellationToken cancellationToken = default)
{
using (var serializedMemory = SerializeCore(message, type))
{
var uncompressed = serializedMemory.Memory;
var compressedBuff = ArrayPool <byte> .Shared.Rent(LZ4Codec.MaximumOutputSize(uncompressed.Length) + HeaderLength);
try
{
//write body, skip header
var compressedLength = LZ4Codec.Encode(
uncompressed.Span,
new Span <byte>(compressedBuff, HeaderLength, compressedBuff.Length - HeaderLength));
//write header
var offset = 0;
SerializerBinary.WriteByte(ref compressedBuff, ref offset, Header);
SerializerBinary.WriteInt32Fixed(ref compressedBuff, ref offset, compressedLength);
SerializerBinary.WriteInt32Fixed(ref compressedBuff, ref offset, uncompressed.Length);
await stream.WriteAsync(compressedBuff, 0, compressedLength + HeaderLength, cancellationToken).ConfigureAwait(false);
}
finally
{
ArrayPool <byte> .Shared.Return(compressedBuff);
}
}
}
public Stream Decompress(Stream stream)
{
var buffer = Utils.ReadAllBytes(stream, 0);
if (buffer.Length < 4)
{
throw new DriverInternalError("Corrupt literal length");
}
var outputLengthBytes = new byte[4];
Buffer.BlockCopy(buffer, 0, outputLengthBytes, 0, 4);
Array.Reverse(outputLengthBytes);
var outputLength = BitConverter.ToInt32(outputLengthBytes, 0);
var outputBuffer = new byte[outputLength];
if (outputLength != 0)
{
var uncompressedSize = LZ4Codec.Decode(buffer, 4, buffer.Length - 4, outputBuffer, 0, outputLength);
if (uncompressedSize < 0)
{
throw new DriverInternalError("LZ4 decoded buffer has a larger size than the expected uncompressed size");
}
if (outputLength != uncompressedSize)
{
throw new DriverInternalError(string.Format("Recorded length is {0} bytes but actual length after decompression is {1} bytes ",
outputLength,
uncompressedSize));
}
}
return(new MemoryStream(outputBuffer, 0, outputLength, false, true));
}
