public async Task <int> ReadAsync_Parallel(int totWorkerThreads, BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_parallel);
}
try
{
int res = 0;
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(
usingMemoryStream ? ms_parallel :
File.Open(tmpFilename_parallel, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
using (var dmp = new CollectionDeserializerAsync <T>(new FIFOWorkerConfig(totWorkerThreads), opts_standard))
await foreach (var i in dmp.DeserializeAsync(s))
{
res ^= i.Item.DoStuff();
}
}
return(res);
}
finally
{
if (usingMemoryStream)
{
ms_parallel.Flush();
ms_parallel.Position = 0;
}
}
}
public async Task FullPipeline_ResultArraysMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemPBLarge[] originalArray = new TestItemPBLarge[totItems];
List <TestItemPBLarge> newArray = new List <TestItemPBLarge>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemPBLarge()
{
f = 1 << f % 32, st = string.Join(",", Enumerable.Range(0, f % 16))
}
}
;
MemoryStream msOrig = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <TestItemPBLarge>(msOrig, cfg))
foreach (TestItemPBLarge item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
var msPB = new MemoryStream(msOrig.ToArray());
using (var ds = new CollectionDeserializerAsync <TestItemPBLarge>(cfg))
await foreach (var item in ds.DeserializeAsync(msPB))
{
newArray.Add(item);
}
Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray());
Assert.AreEqual(originalArray.Select(t => t.st).ToArray(), newArray.Select(t => t.st).ToArray());
}
public async Task DeserializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemPB[] originalArray = new TestItemPB[totItems];
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemPB()
{
f = (byte)(f % 32)
}
}
;
MemoryStream msOrig = new MemoryStream();
for (int i = 0; i < originalArray.Length; i++)
{
TestItemPB item = originalArray[i];
Serializer.SerializeWithLengthPrefix(msOrig, item, ProtoBuf.PrefixStyle.Base128, 1);
}
List <TestItemPB> newArray = new List <TestItemPB>();
var msPB = new MemoryStream(msOrig.ToArray());
using (var ds = new CollectionDeserializerAsync <TestItemPB>(cfg))
await foreach (TestItemPB item in ds.DeserializeAsync(msPB))
{
newArray.Add(item);
}
Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray());
}
public async Task DeserializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
Frame <TestItemMP>[] originalArray = new Frame <TestItemMP> [totItems];
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new Frame <TestItemMP>(2, new TestItemMP()
{
f = (byte)(f % 32)
});
}
List <byte> originalArraySerialized = new List <byte>();
for (int i = 0; i < originalArray.Length; i++)
{
Frame <TestItemMP> item = originalArray[i];
originalArraySerialized.AddRange(MessagePackSerializer.Serialize(item, cancellationToken: CancellationToken.None));
}
List <TestItemMP> newArray = new List <TestItemMP>();
MemoryStream ms = new MemoryStream(originalArraySerialized.ToArray());
using (var deser = new CollectionDeserializerAsync <TestItemMP>(cfg))
await foreach (var item in deser.DeserializeAsync(ms))
{
newArray.Add(item.Item);
}
Assert.AreEqual(originalArray.Select(f => f.Item.f).ToArray(), newArray.Select(t => t.f).ToArray());
}
public static async IAsyncEnumerable <X> New_UnknownLengthArray_ReadAsync(string fileName)
{
using var stream = File.OpenRead(fileName);
using var ds = new CollectionDeserializerAsync <X>(maxConcurrentTasks: 2);
await foreach (var item in ds.DeserializeAsync(stream))
{
yield return(item);
}
}
public void DeserializeAsync_InvalidBodyLengthThrows()
{
var cfg = GetConfig(1, 1);
byte[] bytes = new byte[] { 1, 1 };
var msPB = new MemoryStream(bytes);
Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
using (var ds = new CollectionDeserializerAsync <TestItemPB>(cfg))
await foreach (TestItemPB item in ds.DeserializeAsync(msPB))
{
;
}
});
}
public async Task FullPipeline_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();
SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);
MemoryStream ms = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(new Frame <SerializedItemVarLength>(item), CancellationToken.None);
}
byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
ms = new MemoryStream(originalArraySerialized.ToArray());
using (var deser = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
await foreach (var item in deser.DeserializeAsync(ms))
{
newArray.Add(item.Item);
}
Assert.AreEqual(originalArray, newArray);
}
public async Task CombinedStackAsync_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();
SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);
MemoryStream msOrig = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(msOrig, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
var msPB = new MemoryStream(msOrig.ToArray());
using (var ds = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
await foreach (var item in ds.DeserializeAsync(msPB))
{
newArray.Add(item);
}
Assert.AreEqual(originalArray, newArray);
}