public async Task <int> ReadAsync_Parallel(int totWorkerThreads, BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_parallel);
}
try
{
int res = 0;
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(
usingMemoryStream ? ms_parallel :
File.Open(tmpFilename_parallel, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
using (var dmp = new CollectionDeserializerAsync <T>(new FIFOWorkerConfig(totWorkerThreads), opts_standard))
await foreach (var i in dmp.DeserializeAsync(s))
{
res ^= i.Item.DoStuff();
}
}
return(res);
}
finally
{
if (usingMemoryStream)
{
ms_parallel.Flush();
ms_parallel.Position = 0;
}
}
}
public void RequestNewBuffer_NeverRunsPastShadowBufferSize(int totalPreFetchBlocks, int shadowBufferSize)
{
int stopReadAt = 5;
int curPos = 0;
int forceStopAt = 100;
var streamMock = new Mock <Stream>(MockBehavior.Loose);
streamMock.Setup(p => p.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
.Returns <byte[], int, int>((a, b, c) => { curPos += c; return(c); });
BufferedStreamReaderConfig config = GetConfig(shadowBufferSize, totalPreFetchBlocks, stopReadAt);
List <byte> resArray = new List <byte>();
SynchronousReader pfh = new SynchronousReader(streamMock.Object, config);
int totRead = 0;
int counter = 0;
do
{
ShadowBufferData destBuffer = pfh.RequestNewBuffer();
pfh.ReturnBuffer(destBuffer);
totRead = destBuffer?.byteCount ?? 0;
counter += totRead;
}while (totRead != 0 && counter < forceStopAt);
Assert.AreEqual(stopReadAt, counter);
Assert.AreEqual(stopReadAt, curPos);
}
public async Task <int> ReadAsync_Baseline(BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_baseline);
}
try
{
int res = 0;
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(usingMemoryStream ? ms_baseline :
File.Open(tmpFilename_baseline, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
using (var streamReader = new MessagePackStreamReader(s, true))
while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
var obj = MessagePackSerializer.Deserialize <Frame <T> >(msgpack, opts_standard, CancellationToken.None).Item;
res ^= obj.DoStuff();
}
}
return(res);
}
finally
{
if (usingMemoryStream)
{
ms_baseline.Flush();
ms_baseline.Position = 0;
}
}
}
public async Task <int> ReadAsync_Parallel(int totWorkerThreads, BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_parallel);
}
var opts = new FrameParallelOptions(
totWorkerThreads, opts_standard.WithResolver(FrameResolverPlusStandarResolver.Instance));
try
{
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(
usingMemoryStream ? ms_parallel :
File.Open(tmpFilename_parallel, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
var obj = await MessagePackSerializer.DeserializeAsync <T_Parallel>(s, opts);
return(obj.DoStuff());
}
}
finally
{
if (usingMemoryStream)
{
ms_parallel.Flush();
ms_parallel.Position = 0;
}
}
}
public async Task <int> ReadAsync_Baseline(BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_baseline);
}
try
{
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(
usingMemoryStream ? ms_baseline :
File.Open(tmpFilename_baseline, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
var obj = await MessagePackSerializer.DeserializeAsync <T_Baseline>(s, opts_standard);
return(obj.DoStuff());
}
}
finally
{
if (usingMemoryStream)
{
ms_baseline.Flush();
ms_baseline.Position = 0;
}
}
}
public Stream ComposeChain(Stream s, BufferedStreamReaderConfig sr_cfg)
{
if (!(s is MemoryStream))
{
itemsToDispose.Add(s);
}
if (sr_cfg != null)
{
itemsToDispose.Add(s = new BufferedStreamReader(s, sr_cfg));
}
return(s);
}
public async Task <int> ReadAsync_Baseline(bool isSmall, int totalPreFetchBlocks, bool usingMemoryStream)
{
BufferedStreamReaderConfig config = totalPreFetchBlocks >= 0 ? new BufferedStreamReaderConfig(totalPreFetchBlocks: totalPreFetchBlocks) : null;
if (isSmall)
{
return(await logic_small.ReadAsync_Baseline(config, usingMemoryStream));
}
else
{
return(await logic_large.ReadAsync_Baseline(config, usingMemoryStream));
}
}
public int Read_Baseline(BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_baseline);
}
try
{
Type t = typeof(T);
int res = 0;
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(usingMemoryStream ? ms_baseline :
File.Open(tmpFilename_baseline, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
do
{
object obj = model.DeserializeWithLengthPrefix(s, null, t, PrefixStyle.Base128, 1);
if (obj is T objT)
{
res ^= objT.DoStuff();
}
else if (obj is null)
{
break;
}
else
{
throw new Exception("unexpected obj type deserialized");
}
} while (true);
}
return(res);
}
finally
{
if (usingMemoryStream)
{
ms_baseline.Flush();
ms_baseline.Position = 0;
}
}
}
public void RequestNewBuffer_Returns0BytesAtEOF(int totalPreFetchBlocks, int shadowBufferSize, int totBytes)
{
byte[] originalBuffer = Enumerable.Range(1, totBytes).Select(f => (byte)f).ToArray();
BufferedStreamReaderConfig config = GetConfig(shadowBufferSize, totalPreFetchBlocks);
SynchronousReader pfh = new SynchronousReader(new MemoryStream(originalBuffer), config);
while (true)
{
ShadowBufferData destBuffer = pfh.RequestNewBuffer();
if (destBuffer == null || destBuffer.byteCount == 0)
{
break;
}
pfh.ReturnBuffer(destBuffer);
}
ShadowBufferData lastDestBuffer = pfh.RequestNewBuffer();
int lastRead = lastDestBuffer?.byteCount ?? 0;
Assert.AreEqual(0, lastRead);
}
public void RequestNewBuffer_CorrectBytesAreReturned(int totalPreFetchBlocks, int shadowBufferSize, int totBytes)
{
byte[] originalBuffer = Enumerable.Range(1, totBytes).Select(f => (byte)f).ToArray();
List <byte> resArray = new List <byte>();
BufferedStreamReaderConfig config = GetConfig(shadowBufferSize, totalPreFetchBlocks);
SynchronousReader pfh = new SynchronousReader(new MemoryStream(originalBuffer), config);
while (true)
{
ShadowBufferData destBuffer = pfh.RequestNewBuffer();
if (destBuffer == null || destBuffer.byteCount == 0)
{
break;
}
resArray.AddRange(destBuffer.buffer.Take(destBuffer.byteCount));
pfh.ReturnBuffer(destBuffer);
}
Assert.AreEqual(originalBuffer, resArray.ToArray());
Assert.DoesNotThrow(pfh.Abort);
}
