public void CompressEntry(FilepathInsideArchive filepathInsideArchive, Stream content, bool leaveOpen)
{
if (this.currentItems.ContainsKey(filepathInsideArchive.PathInsideArchive))
{
throw new ArgumentException($"Item '{filepathInsideArchive.PathInsideArchive}' is not already existed in the archive.");
}
BPKG_FTE FileTableEntry = new BPKG_FTE();
FileTableEntry.FilePathLength = filepathInsideArchive.PathInsideArchive.Length;
FileTableEntry.FilePath = filepathInsideArchive.PathInsideArchive;
FileTableEntry.Unknown_001 = 2;
FileTableEntry.IsCompressed = (this.CompressionLevel != CompressionLevel.None);
FileTableEntry.IsEncrypted = true;
FileTableEntry.Unknown_002 = 0;
Stream contentStream = this.CreateTempStream(this.TemporaryFolder);
MemoryStream memoryStream;
using (RecyclableMemoryStream tmp = new RecyclableMemoryStream())
if (filepathInsideArchive.PathInsideArchive.EndsWith(".xml", StringComparison.OrdinalIgnoreCase) || filepathInsideArchive.PathInsideArchive.EndsWith(".x16", StringComparison.OrdinalIgnoreCase))
{
// encode bxml
BXML bxml = new BXML(BXML.XOR_KEY_Data);
BXML.Convert(content, bxml.DetectType(content), tmp, BXML_TYPE.BXML_BINARY);
// FileTableEntry.FileDataOffset = (int)mosFiles.BaseStream.Position;
FileTableEntry.FileDataSizeUnpacked = (int)tmp.Length;
memoryStream = this.Pack(tmp.GetBuffer(), tmp.Length, FileTableEntry.FileDataSizeUnpacked, out FileTableEntry.FileDataSizeSheared, out FileTableEntry.FileDataSizeStored, FileTableEntry.IsEncrypted, FileTableEntry.IsCompressed, this.CompressionLevel);
}
else
{
// compress raw
// FileTableEntry.FileDataOffset = (int)mosFiles.BaseStream.Position;
CommonMethods.StreamCopyChunk(content, tmp, 1024);
FileTableEntry.FileDataSizeUnpacked = (int)content.Length;
memoryStream = this.Pack(tmp.GetBuffer(), tmp.Length, FileTableEntry.FileDataSizeUnpacked, out FileTableEntry.FileDataSizeSheared, out FileTableEntry.FileDataSizeStored, FileTableEntry.IsEncrypted, FileTableEntry.IsCompressed, this.CompressionLevel);
}
contentStream.Write(memoryStream.GetBuffer(), 0, (int)memoryStream.Length);
contentStream.Flush();
memoryStream.Dispose();
memoryStream = null;
if (!leaveOpen)
{
content.Dispose();
}
// FileTableEntry.Padding = new byte[60];
Entry entry = Entry.FromStruct(FileTableEntry);
this.currentItems.Add(FileTableEntry.FilePath, entry);
this.currentContents.Add(entry, contentStream);
}
public async Task <ArraySegment <byte> > GetBytesAsync(CancellationToken cancellationToken)
{
var length = _underlying.Length;
if (length >= int.MaxValue)
{
throw new OverflowException("Streams cannot exceed 2GB");
}
switch (_underlying)
{
case RecyclableMemoryStream underlyingRecMemoryStream:
return(new ArraySegment <byte>(underlyingRecMemoryStream.GetBuffer(), 0,
(int)length));
case MemoryStream underlyingMemoryStream:
if (underlyingMemoryStream.TryGetBuffer(out var underlyingBuffer))
{
return(underlyingBuffer);
}
break;
}
if (_copy == null)
{
_copy = new RecyclableMemoryStream(Mgr, "StreamSource: IBytesSource", (int)length);
await _underlying.CopyToAsync(_copy, 81920, cancellationToken);
}
return(new ArraySegment <byte>(_copy.GetBuffer(), 0,
(int)_copy.Length));
}
/// <summary>
/// Sends data asynchronously
/// </summary>
public void Write(RecyclableMemoryStream stream, Action onBufferFlush)
{
Interlocked.Exchange(ref _writeFlushCallback, onBufferFlush);
if (_isClosing)
{
OnError(new SocketException((int)SocketError.Shutdown));
OnWriteFlushed();
return;
}
if (_sendSocketEvent != null)
{
_sendSocketEvent.BufferList = stream.GetBufferList();
var isWritePending = false;
try
{
isWritePending = _socket.SendAsync(_sendSocketEvent);
}
catch (Exception ex)
{
OnError(ex);
}
if (!isWritePending)
{
OnSendCompleted(this, _sendSocketEvent);
}
}
else
{
var length = (int)stream.Length;
_socketStream.BeginWrite(stream.GetBuffer(), 0, length, OnSendStreamCallback, null);
}
}
private RecyclableMemoryStream Pack(byte[] buffer, long bufferlength, int sizeUnpacked, out int sizeSheared, out int sizeStored, bool encrypt, bool compress, CompressionLevel compressionLevel)
{
RecyclableMemoryStream result = null;
sizeSheared = sizeUnpacked;
sizeStored = sizeSheared;
if (compress && (((int)compressionLevel) != 0))
{
result = Inflate(buffer, bufferlength, sizeUnpacked, out sizeSheared, compressionLevel);
sizeStored = sizeSheared;
}
if (encrypt)
{
if (result == null)
{
result = Encrypt(buffer, bufferlength, sizeSheared, out sizeStored);
}
else
{
result = Encrypt(result.GetBuffer(), (int)result.Length, sizeSheared, out sizeStored);
}
}
return(result);
}
private RecyclableMemoryStream Encrypt(byte[] buffer, long bufferlength, int size, out int sizePadded)
{
sizePadded = size + (RijndaelStream.AES_KEY.Length - (size % RijndaelStream.AES_KEY.Length));
RecyclableMemoryStream result = new RecyclableMemoryStream();
result.SetLength(sizePadded);
byte[] temp = new byte[sizePadded];
unsafe
{
fixed(byte *source = buffer, dest = temp)
{
for (int i = 0; i < bufferlength; i++)
{
dest[i] = source[i];
}
using (Rijndael aes = Rijndael.Create())
{
aes.Mode = CipherMode.ECB;
using (ICryptoTransform encrypt = aes.CreateEncryptor(RijndaelStream.RAW_AES_KEY, new byte[16]))
encrypt.TransformBlock(temp, 0, sizePadded, result.GetBuffer(), 0);
}
}
}
result.Position = 0;
return(result);
}
public static byte[] GetBufferWorkaround(this MemoryStream stream)
{
RecyclableMemoryStream memoryStream = stream as RecyclableMemoryStream;
if (memoryStream != null)
{
return(memoryStream.GetBuffer());
}
return(stream.GetBuffer());
}
/// <summary>
/// Sends data asynchronously
/// </summary>
public void Write(RecyclableMemoryStream stream, Action onBufferFlush)
{
Interlocked.Exchange(ref _writeFlushCallback, onBufferFlush);
if (_isClosing)
{
OnError(new SocketException((int)SocketError.Shutdown));
OnWriteFlushed();
return;
}
if (_sendSocketEvent != null)
{
_sendSocketEvent.BufferList = stream.GetBufferList();
var isWritePending = false;
try
{
isWritePending = _socket.SendAsync(_sendSocketEvent);
}
catch (ObjectDisposedException)
{
OnError(null, SocketError.NotConnected);
}
catch (NullReferenceException)
{
// Mono can throw a NRE when being disposed concurrently
// https://github.com/mono/mono/blob/b190f213a364a2793cc573e1bd9fae8be72296e4/mcs/class/System/System.Net.Sockets/SocketAsyncEventArgs.cs#L184-L185
// https://github.com/mono/mono/blob/b190f213a364a2793cc573e1bd9fae8be72296e4/mcs/class/System/System.Net.Sockets/Socket.cs#L2477-L2478
OnError(null, SocketError.NotConnected);
}
catch (Exception ex)
{
OnError(ex);
}
if (!isWritePending)
{
OnSendCompleted(this, _sendSocketEvent);
}
}
else
{
var length = (int)stream.Length;
try
{
_socketStream
.WriteAsync(stream.GetBuffer(), 0, length)
.ContinueWith(OnSendStreamCallback, TaskContinuationOptions.ExecuteSynchronously);
}
catch (Exception ex)
{
HandleStreamException(ex);
}
}
}
/// <summary>
/// Gets the image using the given identifier.
/// </summary>
/// <param name="id">
/// The value identifying the image to fetch.
/// </param>
/// <returns>
/// The <see cref="System.Byte"/> array containing the image data.
/// </returns>
public virtual async Task <byte[]> GetImage(object id)
{
Uri uri = new Uri(id.ToString());
RemoteFile remoteFile = new RemoteFile(uri)
{
MaxDownloadSize = int.Parse(this.Settings["MaxBytes"]),
TimeoutLength = int.Parse(this.Settings["Timeout"])
};
// Check for optional user agesnt.
if (this.Settings.ContainsKey("Useragent"))
{
if (!string.IsNullOrWhiteSpace(this.Settings["Useragent"]))
{
remoteFile.UserAgent = this.Settings["Useragent"];
}
}
byte[] buffer;
// Prevent response blocking.
WebResponse webResponse = await remoteFile.GetWebResponseAsync().ConfigureAwait(false);
using (RecyclableMemoryStream memoryStream = new RecyclableMemoryStream(MemoryStreamPool.Shared))
{
using (WebResponse response = webResponse)
{
using (Stream responseStream = response.GetResponseStream())
{
if (responseStream != null)
{
responseStream.CopyTo(memoryStream);
// Reset the position of the stream to ensure we're reading the correct part.
memoryStream.Position = 0;
buffer = memoryStream.GetBuffer();
}
else
{
throw new HttpException((int)HttpStatusCode.NotFound, $"No image exists at {uri}");
}
}
}
}
return(buffer);
}
public static byte[] FromFile(string basePath, string hashCode)
{
var filePath = Path.Combine(basePath, $"objects/{hashCode.Substring(0, 2)}/{hashCode.Substring(2)}");
using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
using (var stream = new DeflateStream(fileStream, CompressionMode.Decompress, false))
using (var memoryStream = new RecyclableMemoryStream(MemoryManager, hashCode))
{
fileStream.Seek(2, SeekOrigin.Begin);
stream.CopyTo(memoryStream);
var result = new byte[memoryStream.Length];
Array.Copy(memoryStream.GetBuffer(), result, memoryStream.Length);
return(result);
}
}
public byte[] GetBuffer()
{
ThrowIfDisposed();
return(m_stream.GetBuffer());
}
public void WriteArchive()
{
this.BaseStream.Position = 0;
if (this.BaseStream.Length != 0)
{
this.BaseStream.SetLength(0);
}
Entry[] entryList = this.currentContents.Keys.ToArray();
Entry currentprocessingEntry;
using (RecyclableMemoryStream rms_header = new RecyclableMemoryStream())
using (BinaryWriter bw_header = new BinaryWriter(rms_header))
{
long virtualcontentoffset = 0;
for (int i = 0; i < entryList.Length; i++)
{
currentprocessingEntry = entryList[i];
currentprocessingEntry.Raw.FileDataOffset = (int)virtualcontentoffset;
virtualcontentoffset += this.currentContents[currentprocessingEntry].Length;
if (this.Is64bit)
{
bw_header.Write((long)currentprocessingEntry.Raw.FilePathLength);
}
else
{
bw_header.Write(currentprocessingEntry.Raw.FilePathLength);
}
bw_header.Write(Encoding.Unicode.GetBytes(currentprocessingEntry.Raw.FilePath));
bw_header.Write(currentprocessingEntry.Raw.Unknown_001);
bw_header.Write(currentprocessingEntry.Raw.IsCompressed);
bw_header.Write(currentprocessingEntry.Raw.IsEncrypted);
bw_header.Write(currentprocessingEntry.Raw.Unknown_002);
if (this.Is64bit)
{
bw_header.Write((long)currentprocessingEntry.Raw.FileDataSizeUnpacked);
}
else
{
bw_header.Write(currentprocessingEntry.Raw.FileDataSizeUnpacked);
}
if (this.Is64bit)
{
bw_header.Write((long)currentprocessingEntry.Raw.FileDataSizeSheared);
}
else
{
bw_header.Write(currentprocessingEntry.Raw.FileDataSizeSheared);
}
if (this.Is64bit)
{
bw_header.Write((long)currentprocessingEntry.Raw.FileDataSizeStored);
}
else
{
bw_header.Write(currentprocessingEntry.Raw.FileDataSizeStored);
}
if (this.Is64bit)
{
bw_header.Write((long)currentprocessingEntry.Raw.FileDataOffset);
}
else
{
bw_header.Write(currentprocessingEntry.Raw.FileDataOffset);
}
bw_header.Write(headerPadding);
}
bw_header.Flush();
BinaryWriter bw = new BinaryWriter(this.BaseStream);
byte[] Signature = new byte[8] {
(byte)'U', (byte)'O', (byte)'S', (byte)'E', (byte)'D', (byte)'A', (byte)'L', (byte)'B'
};
bw.Write(Signature);
// Writing Version
bw.Write((int)2);
// Write Unknown_001
bw.Write(new byte[5] {
0, 0, 0, 0, 0
});
// Write FileDataSizePacked
if (this.Is64bit)
{
bw.Write(virtualcontentoffset);
bw.Write((long)this.ItemCount);
}
else
{
bw.Write((int)virtualcontentoffset);
bw.Write(this.ItemCount);
}
// Write IsCompressed
bool isCompressed = (this.CompressionLevel != CompressionLevel.None);
bw.Write(isCompressed);
// Write IsEncrypted
bw.Write(true);
// Write Unknown_002
bw.Write(new byte[62]);
int FileTableSizeUnpacked = (int)rms_header.Length;
int FileTableSizeSheared = FileTableSizeUnpacked;
int FileTableSizePacked = FileTableSizeUnpacked;
using (var packedHeader = this.Pack(rms_header.GetBuffer(), rms_header.Length, FileTableSizeUnpacked, out FileTableSizeSheared, out FileTableSizePacked, true, isCompressed, this.CompressionLevel))
{
if (this.Is64bit)
{
bw.Write((long)FileTableSizePacked);
}
else
{
bw.Write(FileTableSizePacked);
}
if (this.Is64bit)
{
bw.Write((long)FileTableSizeUnpacked);
}
else
{
bw.Write(FileTableSizeUnpacked);
}
bw.Write(packedHeader.GetBuffer(), 0, (int)packedHeader.Length);
}
int OffsetGlobal = (int)this.BaseStream.Position + (this.Is64bit ? 8 : 4);
if (this.Is64bit)
{
bw.Write((long)OffsetGlobal);
}
else
{
bw.Write(OffsetGlobal);
}
bw.Flush();
Stream currentProcessingStream;
for (int i = 0; i < entryList.Length; i++)
{
currentprocessingEntry = entryList[i];
currentProcessingStream = this.currentContents[currentprocessingEntry];
currentProcessingStream.Seek(0, SeekOrigin.Begin);
CommonMethods.StreamCopyChunk(currentProcessingStream, bw.BaseStream, 4096);
}
bw.BaseStream.Flush();
}
}
public void GetBufferAdjustsLargePoolFreeSize()
{
var stream = this.GetDefaultStream();
var memMgr = stream.MemoryManager;
var bufferLength = stream.MemoryManager.BlockSize * 4;
var buffer = this.GetRandomBuffer(bufferLength);
stream.Write(buffer, 0, buffer.Length);
var newBuffer = stream.GetBuffer();
stream.Dispose();
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(newBuffer.Length));
var newStream = new RecyclableMemoryStream(memMgr);
newStream.Write(buffer, 0, buffer.Length);
var newBuffer2 = newStream.GetBuffer();
Assert.That(newBuffer2.Length, Is.EqualTo(newBuffer.Length));
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(0));
}