public async Task TestReadFromStream()
{
var options = new FileBufferOptions(100, 10);
await using var buffers = new FileBuffers(options);
using var memStream = new MemoryStream();
var data = new byte[150];
for (byte i = 0; i < data.Length; i++)
{
data[i] = i;
}
memStream.Write(data, 0, data.Length);
var id = new BufferId(Guid.NewGuid(), "*");
await using var buffer = await buffers.AddForMememoryStream(id, memStream).NoSync();
var page = UseAndReturn.That(await buffer.GetPageAsync(0).ConfigureAwait(false));
AssertEqual(1, page.Data[1]);
AssertEqual(100, page.Data.Length);
await page.DisposeAsync().NoSync();
AssertEqual(1, buffers.Pool.Count);
page = UseAndReturn.That(await buffer.GetPageAsync(0).ConfigureAwait(false));
AssertEqual(0L, page.Data.Number);
AssertEqual(1, buffer.NumberOfBufferedPages);
await page.DisposeAsync().NoSync();
}
public static async Task Main()
{
await using var buffers = new FileBuffers(new FileBufferOptions());
var id = new BufferId(Guid.NewGuid(), "*");
var buffer = await buffers.AddForFile(id, "d:\\temp\\2.abc").NoSync();
for (var i = 0; i < buffer.PageCount; i++)
{
await using var p1 = UseAndReturn.That(await buffer.GetPageAsync(i).NoSync());
Console.WriteLine("Data: " + p1.Data[0].ToString(CultureInfo.CurrentCulture));
}
}
public async Task TestPageCaching()
{
var options = new FileBufferOptions(10, 5);
await using var buffers = new FileBuffers(options);
var memStream = new MemoryStream();
var data = new byte[150];
for (byte i = 0; i < data.Length; i++)
{
data[i] = i;
}
memStream.Write(data, 0, data.Length);
var id = new BufferId(Guid.NewGuid(), "*");
await using var buffer = await buffers.AddForMememoryStream(id, memStream).NoSync();
await using var page1 = await buffer.GetPageAsync(0).NoSync();
await using var page2 = await buffer.GetPageAsync(1).NoSync();
await using var page3 = await buffer.GetPageAsync(2).NoSync();
await using var page4 = await buffer.GetPageAsync(3).NoSync();
await using (var page5 = await buffer.GetPageAsync(4).NoSync()) {
AssertEqual(0L, page1.Number);
AssertEqual(1L, page2.Number);
AssertEqual(2L, page3.Number);
AssertEqual(3L, page4.Number);
AssertEqual(4L, page5.Number);
AssertEqual(5, buffer.NumberOfBufferedPages);
await using var page6 = await buffer.GetPageAsync(5).ConfigureAwait(false);
AssertEqual(5L, page6.Number);
AssertEqual(6, buffer.NumberOfBufferedPages);
await page6.UnPin().NoSync();
}
AssertEqual(5, buffer.NumberOfBufferedPages);
}
/// <summary>
/// 写入数据
/// </summary>
/// <param name="buffer"></param>
/// <param name="bufferStart"></param>
private void write(ref FileBuffers buffer, byte *bufferStart)
{
if (*(int *)bufferStart >= Config.MinCompressSize)
{
if (AutoCSer.IO.Compression.DeflateCompressor.Get(buffer.Buffer.Buffer, buffer.Buffer.StartIndex + PacketHeaderSize, *(int *)bufferStart, ref buffer.CompressionBuffer, ref buffer.CompressionData, PacketHeaderSize + sizeof(int), PacketHeaderSize + sizeof(int)))
{
writeCompression(ref buffer.CompressionData, bufferStart);
buffer.CompressionBuffer.TryFree();
return;
}
buffer.CompressionBuffer.TryFree();
}
int size = *(int *)bufferStart + PacketHeaderSize;
dataFileStream.Write(buffer.Buffer.Buffer, buffer.Buffer.StartIndex, size);
dataFileLength += size;
setIndex();
}
/// <summary>
/// 写入数据
/// </summary>
private unsafe void write()
{
FileBuffers buffer = default(FileBuffers);
int bigBufferSize;
bool isCopyBuffer, isNeedDispose = false;
try
{
bufferPool.Get(ref buffer.Buffer);
DateTime callbackTime = Date.NowTime.Now;
fixed(byte *bufferFixed = buffer.Buffer.Buffer)
{
byte *bufferStart = bufferFixed + buffer.Buffer.StartIndex;
GETQUEUE:
//Thread.Sleep(0);
Buffer head = bufferQueue.GetClear();
if (head != null)
{
int index = PacketHeaderSize, dataCount = 0;
Buffer data = head;
do
{
data.Identity = identity;
int dataSize = data.Data.GetSerializeBufferSize(out isCopyBuffer);
CHECKSIZE:
if (dataSize + index <= buffer.Buffer.Length)
{
data.Data.SerializeBuffer(bufferStart + index);
if (isCopyBuffer)
{
data.Data.SerializeBuffer(ref buffer.Buffer, index);
}
index += dataSize;
++dataCount;
++identity;
}
else if (dataCount != 0)
{
*(int *)bufferStart = index - PacketHeaderSize;
*(int *)(bufferStart + sizeof(int)) = dataCount;
write(ref buffer, bufferStart);
index = PacketHeaderSize;
dataCount = 0;
goto CHECKSIZE;
}
else
{
#region 写入大数据
++identity;
if (bigBuffer.Length < (bigBufferSize = dataSize + PacketHeaderSize))
{
bigBuffer = new byte[Math.Max(bigBufferSize, bigBuffer.Length << 1)];
fixed(byte *bigBufferFixed = bigBuffer)
{
*(int *)bigBufferFixed = dataSize;
*(int *)(bigBufferFixed + sizeof(int)) = 1;
data.Data.SerializeBuffer(bigBufferFixed + PacketHeaderSize, bigBuffer);
if (dataSize >= Config.MinCompressSize)
{
if (AutoCSer.IO.Compression.DeflateCompressor.Get(bigBuffer, PacketHeaderSize, dataSize, ref buffer.CompressionBuffer, ref buffer.CompressionData, PacketHeaderSize + sizeof(int), PacketHeaderSize + sizeof(int)))
{
writeCompression(ref buffer.CompressionData, bigBufferFixed);
buffer.CompressionBuffer.TryFree();
goto CHECKIDENTITY;
}
buffer.CompressionBuffer.TryFree();
}
}
dataFileStream.Write(bigBuffer, 0, bigBufferSize);
dataFileLength += bigBufferSize;
setIndex();
#endregion
}
CHECKIDENTITY:
if (((uint)identity & (DataCountPerFile - 1)) == 0)
{
#region 切换数据文件
if (dataCount != 0)
{
*(int *)bufferStart = index - PacketHeaderSize;
*(int *)(bufferStart + sizeof(int)) = dataCount;
write(ref buffer, bufferStart);
index = PacketHeaderSize;
dataCount = 0;
}
dataFileStream.Flush(true);
dataFileStream.Dispose();
dataFileStream = null;
writeState(ref buffer.Buffer);
writeIndex(ref buffer.Buffer);
FileInfo dataFileInfo = new FileInfo(dataFileName);
if (dataFileInfo.Exists)
{
if (dataFileInfo.Length == 0)
{
dataFileInfo.Delete();
}
else
{
AutoCSer.IO.File.MoveBak(dataFileInfo.FullName);
}
}
dataFileStream = new FileStream(dataFileInfo.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.Read, bufferPool.Size, FileOptions.None);
dataFileLength = 0;
#endregion
}
else if (dataCount == MaxPacketCount)
{
*(int *)bufferStart = index - PacketHeaderSize;
*(int *)(bufferStart + sizeof(int)) = MaxPacketCount;
write(ref buffer, bufferStart);
index = PacketHeaderSize;
dataCount = 0;
}
if (data.LinkNext == null)
{
if (bufferQueue.IsEmpty || callbackTime != Date.NowTime.Now)
break; }
data.LinkNext = bufferQueue.GetClear();
}
data = data.LinkNext;
}while (true);
if (dataCount != 0)
{
*(int *)bufferStart = index - PacketHeaderSize;
*(int *)(bufferStart + sizeof(int)) = dataCount;
write(ref buffer, bufferStart);
}
dataFileStream.Flush(true);
writeState(ref buffer.Buffer);
if (ReadCount == 0)
{
do
{
head.Callback(ref head);
}while (head != null);
}
else
{
new QueueTaskThread.Append(head).AddQueueTaskLinkThread();
}
callbackTime = Date.NowTime.Now;
}
if (this.isNeedDispose != 0)
{
isNeedDispose = true;
return;
}
if (!bufferQueue.IsEmpty)
{
goto GETQUEUE;
}
Interlocked.Exchange(ref isWrite, 0);
if (!bufferQueue.IsEmpty && Interlocked.CompareExchange(ref isWrite, 1, 0) == 0)
{
goto GETQUEUE;
}
}
}
catch (Exception error)
{
isNeedDispose = true;
AutoCSer.Log.Pub.Log.Add(Log.LogType.Fatal, error);
}
finally
{
buffer.Free();
if (isNeedDispose)
{
Dispose();
}
}
}
