public async Task <int> ReadAsync_Parallel(int totWorkerThreads, BufferedStreamReaderConfig config_r, bool usingMemoryStream) { if (!usingMemoryStream) { FileHelper.FlushFileCache(tmpFilename_parallel); } try { int res = 0; using (StreamChain sc1 = new StreamChain()) { Stream s = sc1.ComposeChain( usingMemoryStream ? ms_parallel : File.Open(tmpFilename_parallel, FileMode.Open, FileAccess.Read, FileShare.None), config_r); using (var dmp = new CollectionDeserializerAsync <T>(new FIFOWorkerConfig(totWorkerThreads), opts_standard)) await foreach (var i in dmp.DeserializeAsync(s)) { res ^= i.Item.DoStuff(); } } return(res); } finally { if (usingMemoryStream) { ms_parallel.Flush(); ms_parallel.Position = 0; } } }
public async Task FullPipeline_ResultArraysMatchesExpected(int totItems, int totThreads, int maxQueuedItems) { var cfg = GetConfig(totThreads, maxQueuedItems); TestItemPBLarge[] originalArray = new TestItemPBLarge[totItems]; List <TestItemPBLarge> newArray = new List <TestItemPBLarge>(); for (int f = 0; f < totItems; f++) { originalArray[f] = new TestItemPBLarge() { f = 1 << f % 32, st = string.Join(",", Enumerable.Range(0, f % 16)) } } ; MemoryStream msOrig = new MemoryStream(); await using (var ser = new CollectionSerializerAsync <TestItemPBLarge>(msOrig, cfg)) foreach (TestItemPBLarge item in originalArray) { await ser.SerializeAsync(item, CancellationToken.None); } var msPB = new MemoryStream(msOrig.ToArray()); using (var ds = new CollectionDeserializerAsync <TestItemPBLarge>(cfg)) await foreach (var item in ds.DeserializeAsync(msPB)) { newArray.Add(item); } Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray()); Assert.AreEqual(originalArray.Select(t => t.st).ToArray(), newArray.Select(t => t.st).ToArray()); }
public async Task DeserializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems) { var cfg = GetConfig(totThreads, maxQueuedItems); TestItemPB[] originalArray = new TestItemPB[totItems]; for (int f = 0; f < totItems; f++) { originalArray[f] = new TestItemPB() { f = (byte)(f % 32) } } ; MemoryStream msOrig = new MemoryStream(); for (int i = 0; i < originalArray.Length; i++) { TestItemPB item = originalArray[i]; Serializer.SerializeWithLengthPrefix(msOrig, item, ProtoBuf.PrefixStyle.Base128, 1); } List <TestItemPB> newArray = new List <TestItemPB>(); var msPB = new MemoryStream(msOrig.ToArray()); using (var ds = new CollectionDeserializerAsync <TestItemPB>(cfg)) await foreach (TestItemPB item in ds.DeserializeAsync(msPB)) { newArray.Add(item); } Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray()); }
public async Task DeserializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems) { var cfg = GetConfig(totThreads, maxQueuedItems); Frame <TestItemMP>[] originalArray = new Frame <TestItemMP> [totItems]; for (int f = 0; f < totItems; f++) { originalArray[f] = new Frame <TestItemMP>(2, new TestItemMP() { f = (byte)(f % 32) }); } List <byte> originalArraySerialized = new List <byte>(); for (int i = 0; i < originalArray.Length; i++) { Frame <TestItemMP> item = originalArray[i]; originalArraySerialized.AddRange(MessagePackSerializer.Serialize(item, cancellationToken: CancellationToken.None)); } List <TestItemMP> newArray = new List <TestItemMP>(); MemoryStream ms = new MemoryStream(originalArraySerialized.ToArray()); using (var deser = new CollectionDeserializerAsync <TestItemMP>(cfg)) await foreach (var item in deser.DeserializeAsync(ms)) { newArray.Add(item.Item); } Assert.AreEqual(originalArray.Select(f => f.Item.f).ToArray(), newArray.Select(t => t.f).ToArray()); }
public static async IAsyncEnumerable <X> New_UnknownLengthArray_ReadAsync(string fileName) { using var stream = File.OpenRead(fileName); using var ds = new CollectionDeserializerAsync <X>(maxConcurrentTasks: 2); await foreach (var item in ds.DeserializeAsync(stream)) { yield return(item); } }
public void DeserializeAsync_InvalidBodyLengthThrows() { var cfg = GetConfig(1, 1); byte[] bytes = new byte[] { 1, 1 }; var msPB = new MemoryStream(bytes); Assert.ThrowsAsync <InvalidOperationException>(async() => { using (var ds = new CollectionDeserializerAsync <TestItemPB>(cfg)) await foreach (TestItemPB item in ds.DeserializeAsync(msPB)) { ; } }); }
public async Task FullPipeline_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength) { var cfg = GetConfig(totThreads, maxQueuedItems); List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>(); SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength); MemoryStream ms = new MemoryStream(); await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(ms, cfg)) foreach (var item in originalArray) { await ser.SerializeAsync(new Frame <SerializedItemVarLength>(item), CancellationToken.None); } byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray(); ms = new MemoryStream(originalArraySerialized.ToArray()); using (var deser = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg)) await foreach (var item in deser.DeserializeAsync(ms)) { newArray.Add(item.Item); } Assert.AreEqual(originalArray, newArray); }
public async Task CombinedStackAsync_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength) { var cfg = GetConfig(totThreads, maxQueuedItems); List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>(); SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength); MemoryStream msOrig = new MemoryStream(); await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(msOrig, cfg)) foreach (var item in originalArray) { await ser.SerializeAsync(item, CancellationToken.None); } var msPB = new MemoryStream(msOrig.ToArray()); using (var ds = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg)) await foreach (var item in ds.DeserializeAsync(msPB)) { newArray.Add(item); } Assert.AreEqual(originalArray, newArray); }