public async Task WriteAsync_Parallel(int totWorkerThreads, BufferedStreamWriterConfig sw_cfg, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_baseline);
}
else
{
ms_parallel.SetLength(ms_parallel.Length);
ms_parallel.Position = 0;
}
FileHelper.FlushFileCache(tmpFilename_parallel);
using (StreamChain sc = new StreamChain())
{
Stream s = sc.ComposeChain(
usingMemoryStream ? ms_parallel :
File.Open(tmpFilename_parallel, FileMode.Create, FileAccess.Write, FileShare.None), sw_cfg);
using (var smp = new CollectionSerializerAsync <T>(s, new FIFOWorkerConfig(totWorkerThreads), new BatchSizeEstimatorConfig(), opts_standard))
foreach (var obj in objArr)
{
await smp.SerializeAsync(new Frame <T>(obj));
}
}
if (usingMemoryStream)
{
ms_parallel.Flush();
ms_parallel.Position = 0;
}
}
public async Task FullPipeline_ResultArraysMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemPBLarge[] originalArray = new TestItemPBLarge[totItems];
List <TestItemPBLarge> newArray = new List <TestItemPBLarge>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemPBLarge()
{
f = 1 << f % 32, st = string.Join(",", Enumerable.Range(0, f % 16))
}
}
;
MemoryStream msOrig = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <TestItemPBLarge>(msOrig, cfg))
foreach (TestItemPBLarge item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
var msPB = new MemoryStream(msOrig.ToArray());
using (var ds = new CollectionDeserializerAsync <TestItemPBLarge>(cfg))
await foreach (var item in ds.DeserializeAsync(msPB))
{
newArray.Add(item);
}
Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray());
Assert.AreEqual(originalArray.Select(t => t.st).ToArray(), newArray.Select(t => t.st).ToArray());
}
public async Task SerializeAsync_DeserializesAsValidArrayWrapperInstance(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemProto[] originalArray = new TestItemProto[totItems];
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemProto()
{
f = (byte)(f + 10)
}
}
;
MemoryStream ms = new MemoryStream();
using (var ser = new CollectionSerializerAsync <TestItemProto>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
byte[] originalArraySerialized = ms.ToArray();
var deserializedArrayWrapper = Serializer.Deserialize <ParallelServices_ArrayWrapper <TestItemProto> >(new MemoryStream(originalArraySerialized));
Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), deserializedArrayWrapper.Array.Select(t => t.f).ToArray());
}
public async Task SerializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemMP[] originalArray = new TestItemMP[totItems];
List <TestItemMP> asyncArray = new List <TestItemMP>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemMP()
{
f = (byte)(1 + f % 32)
}
}
;
MemoryStream ms = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <TestItemMP>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
using (var streamReader = new MessagePackStreamReader(new MemoryStream(originalArraySerialized)))
while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
asyncArray.Add(MessagePackSerializer.Deserialize <Frame <TestItemMP> >(msgpack, cancellationToken: CancellationToken.None));
}
Assert.AreEqual(originalArray.Select(f => f.f).ToArray(), asyncArray.Select(t => t.f).ToArray());
}
public static async Task New_UnknownLengthArray_WriteAsync(string fileName)
{
var arr = GetSampleArray();
using var stream = File.Create(fileName);
await using var ser = new CollectionSerializerAsync <X>(stream, maxConcurrentTasks: 2);
foreach (var item in arr)
{
await ser.SerializeAsync(item);
}
}
public static async Task New_UnknownLengthArray_WriteAsync(string fileName)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
IEnumerable <X> arr = GetSampleArray();
using var stream = File.Create(fileName);
await using var ser = new CollectionSerializerAsync <X>(stream, new FIFOWorkerConfig(maxConcurrentTasks: 2));
foreach (var item in arr)
{
await ser.SerializeAsync(item);
}
}
public async Task SerializeAsync_FrameHeadersAreCorrect(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemMP[] originalArray = new TestItemMP[totItems];
List <byte> syncSerializedWithFrameData = new List <byte>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemMP()
{
f = (byte)(1 + f % 32)
}
}
;
MemoryStream ms = new MemoryStream();
foreach (var item in originalArray)
{
int bodylen = MessagePackSerializer.Serialize(item, MessagePackSerializerOptions.Standard).Length;
//bodylen = bodylen | (1 << 24); // (A) trick to force writing Int32
Frame <TestItemMP> framedItem = new Frame <TestItemMP>(bodylen, item);
syncSerializedWithFrameData.AddRange(MessagePackSerializer.Serialize(framedItem, cancellationToken: CancellationToken.None));
}
//for (int i = 0; i < syncSerializedWithFrameData.Count - 1; i++)
//{
// // undo (A)
// if (syncSerializedWithFrameData[i] == MessagePackCode.UInt32 && syncSerializedWithFrameData[i + 1] == 1)
// syncSerializedWithFrameData[i + 1] = 0;
//}
await using (var ser = new CollectionSerializerAsync <TestItemMP>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
Assert.AreEqual(syncSerializedWithFrameData.ToArray(), originalArraySerialized);
}
public async Task FullPipeline_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();
SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);
MemoryStream ms = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(new Frame <SerializedItemVarLength>(item), CancellationToken.None);
}
byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
ms = new MemoryStream(originalArraySerialized.ToArray());
using (var deser = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
await foreach (var item in deser.DeserializeAsync(ms))
{
newArray.Add(item.Item);
}
Assert.AreEqual(originalArray, newArray);
}
public async Task SerializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemPB[] originalArray = new TestItemPB[totItems];
List <TestItemPB> deserializedArray = new List <TestItemPB>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemPB()
{
f = (byte)(1 + f % 32)
}
}
;
MemoryStream ms = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <TestItemPB>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
using (var msPB = new MemoryStream(ms.ToArray()))
{
while (true)
{
var obj = ProtoBuf.Serializer.DeserializeWithLengthPrefix <TestItemPB>(msPB, ProtoBuf.PrefixStyle.Base128, 1);
if (obj is null)
{
break;
}
deserializedArray.Add(obj);
}
}
Assert.AreEqual(originalArray.Select(f => f.f).ToArray(), deserializedArray.Select(t => t.f).ToArray());
}
public async Task CombinedStackAsync_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();
SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);
MemoryStream msOrig = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(msOrig, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
var msPB = new MemoryStream(msOrig.ToArray());
using (var ds = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
await foreach (var item in ds.DeserializeAsync(msPB))
{
newArray.Add(item);
}
Assert.AreEqual(originalArray, newArray);
}