public async Task WriteAsync_Parallel(int totWorkerThreads, BufferedStreamWriterConfig sw_cfg, bool usingMemoryStream)
 {
     if (!usingMemoryStream)
     {
         FileHelper.FlushFileCache(tmpFilename_baseline);
     }
     else
     {
         ms_parallel.SetLength(ms_parallel.Length);
         ms_parallel.Position = 0;
     }
     FileHelper.FlushFileCache(tmpFilename_parallel);
     using (StreamChain sc = new StreamChain())
     {
         Stream s = sc.ComposeChain(
             usingMemoryStream ? ms_parallel :
             File.Open(tmpFilename_parallel, FileMode.Create, FileAccess.Write, FileShare.None), sw_cfg);
         using (var smp = new CollectionSerializerAsync <T>(s, new FIFOWorkerConfig(totWorkerThreads), new BatchSizeEstimatorConfig(), opts_standard))
             foreach (var obj in objArr)
             {
                 await smp.SerializeAsync(new Frame <T>(obj));
             }
     }
     if (usingMemoryStream)
     {
         ms_parallel.Flush();
         ms_parallel.Position = 0;
     }
 }
        public async Task FullPipeline_ResultArraysMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);

            TestItemPBLarge[]      originalArray = new TestItemPBLarge[totItems];
            List <TestItemPBLarge> newArray      = new List <TestItemPBLarge>();

            for (int f = 0; f < totItems; f++)
            {
                originalArray[f] = new TestItemPBLarge()
                {
                    f = 1 << f % 32, st = string.Join(",", Enumerable.Range(0, f % 16))
                }
            }
            ;

            MemoryStream msOrig = new MemoryStream();

            await using (var ser = new CollectionSerializerAsync <TestItemPBLarge>(msOrig, cfg))
                foreach (TestItemPBLarge item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }
            var msPB = new MemoryStream(msOrig.ToArray());

            using (var ds = new CollectionDeserializerAsync <TestItemPBLarge>(cfg))
                await foreach (var item in ds.DeserializeAsync(msPB))
                {
                    newArray.Add(item);
                }

            Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), newArray.Select(t => t.f).ToArray());
            Assert.AreEqual(originalArray.Select(t => t.st).ToArray(), newArray.Select(t => t.st).ToArray());
        }
        public async Task SerializeAsync_DeserializesAsValidArrayWrapperInstance(int totItems, int totThreads, int maxQueuedItems)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);

            TestItemProto[] originalArray = new TestItemProto[totItems];
            for (int f = 0; f < totItems; f++)
            {
                originalArray[f] = new TestItemProto()
                {
                    f = (byte)(f + 10)
                }
            }
            ;
            MemoryStream ms = new MemoryStream();

            using (var ser = new CollectionSerializerAsync <TestItemProto>(ms, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }

            byte[] originalArraySerialized  = ms.ToArray();
            var    deserializedArrayWrapper = Serializer.Deserialize <ParallelServices_ArrayWrapper <TestItemProto> >(new MemoryStream(originalArraySerialized));

            Assert.AreEqual(originalArray.Select(t => t.f).ToArray(), deserializedArrayWrapper.Array.Select(t => t.f).ToArray());
        }
Ejemplo n.º 4
0
        public async Task SerializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);

            TestItemMP[]      originalArray = new TestItemMP[totItems];
            List <TestItemMP> asyncArray    = new List <TestItemMP>();

            for (int f = 0; f < totItems; f++)
            {
                originalArray[f] = new TestItemMP()
                {
                    f = (byte)(1 + f % 32)
                }
            }
            ;
            MemoryStream ms = new MemoryStream();

            await using (var ser = new CollectionSerializerAsync <TestItemMP>(ms, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }

            byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
            using (var streamReader = new MessagePackStreamReader(new MemoryStream(originalArraySerialized)))
                while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
                {
                    asyncArray.Add(MessagePackSerializer.Deserialize <Frame <TestItemMP> >(msgpack, cancellationToken: CancellationToken.None));
                }
            Assert.AreEqual(originalArray.Select(f => f.f).ToArray(), asyncArray.Select(t => t.f).ToArray());
        }
        public static async Task New_UnknownLengthArray_WriteAsync(string fileName)
        {
            var arr = GetSampleArray();

            using var stream    = File.Create(fileName);
            await using var ser = new CollectionSerializerAsync <X>(stream, maxConcurrentTasks: 2);
            foreach (var item in arr)
            {
                await ser.SerializeAsync(item);
            }
        }
Ejemplo n.º 6
0
        public static async Task New_UnknownLengthArray_WriteAsync(string fileName)
        {
            if (File.Exists(fileName))
            {
                File.Delete(fileName);
            }
            IEnumerable <X> arr = GetSampleArray();

            using var stream    = File.Create(fileName);
            await using var ser = new CollectionSerializerAsync <X>(stream, new FIFOWorkerConfig(maxConcurrentTasks: 2));
            foreach (var item in arr)
            {
                await ser.SerializeAsync(item);
            }
        }
Ejemplo n.º 7
0
        public async Task SerializeAsync_FrameHeadersAreCorrect(int totItems, int totThreads, int maxQueuedItems)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);

            TestItemMP[] originalArray = new TestItemMP[totItems];
            List <byte>  syncSerializedWithFrameData = new List <byte>();

            for (int f = 0; f < totItems; f++)
            {
                originalArray[f] = new TestItemMP()
                {
                    f = (byte)(1 + f % 32)
                }
            }
            ;
            MemoryStream ms = new MemoryStream();

            foreach (var item in originalArray)
            {
                int bodylen = MessagePackSerializer.Serialize(item, MessagePackSerializerOptions.Standard).Length;
                //bodylen = bodylen | (1 << 24); // (A) trick to force writing Int32
                Frame <TestItemMP> framedItem = new Frame <TestItemMP>(bodylen, item);
                syncSerializedWithFrameData.AddRange(MessagePackSerializer.Serialize(framedItem, cancellationToken: CancellationToken.None));
            }
            //for (int i = 0; i < syncSerializedWithFrameData.Count - 1; i++)
            //{
            //    // undo (A)
            //    if (syncSerializedWithFrameData[i] == MessagePackCode.UInt32 && syncSerializedWithFrameData[i + 1] == 1)
            //        syncSerializedWithFrameData[i + 1] = 0;
            //}

            await using (var ser = new CollectionSerializerAsync <TestItemMP>(ms, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }

            byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
            Assert.AreEqual(syncSerializedWithFrameData.ToArray(), originalArraySerialized);
        }
Ejemplo n.º 8
0
        public async Task FullPipeline_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);
            List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();

            SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);
            MemoryStream ms = new MemoryStream();

            await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(ms, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(new Frame <SerializedItemVarLength>(item), CancellationToken.None);
                }
            byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
            ms = new MemoryStream(originalArraySerialized.ToArray());
            using (var deser = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
                await foreach (var item in deser.DeserializeAsync(ms))
                {
                    newArray.Add(item.Item);
                }

            Assert.AreEqual(originalArray, newArray);
        }
        public async Task SerializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);

            TestItemPB[]      originalArray     = new TestItemPB[totItems];
            List <TestItemPB> deserializedArray = new List <TestItemPB>();

            for (int f = 0; f < totItems; f++)
            {
                originalArray[f] = new TestItemPB()
                {
                    f = (byte)(1 + f % 32)
                }
            }
            ;
            MemoryStream ms = new MemoryStream();

            await using (var ser = new CollectionSerializerAsync <TestItemPB>(ms, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }

            using (var msPB = new MemoryStream(ms.ToArray()))
            {
                while (true)
                {
                    var obj = ProtoBuf.Serializer.DeserializeWithLengthPrefix <TestItemPB>(msPB, ProtoBuf.PrefixStyle.Base128, 1);
                    if (obj is null)
                    {
                        break;
                    }
                    deserializedArray.Add(obj);
                }
            }
            Assert.AreEqual(originalArray.Select(f => f.f).ToArray(), deserializedArray.Select(t => t.f).ToArray());
        }
        public async Task CombinedStackAsync_MultipleSizedObjects_ResultMatchesInput(int totItems, int totThreads, int maxQueuedItems, int innerLength)
        {
            var cfg = GetConfig(totThreads, maxQueuedItems);
            List <SerializedItemVarLength> newArray = new List <SerializedItemVarLength>();

            SerializedItemVarLength[] originalArray = GenerateObjectArrayVariableSize(totItems, innerLength);

            MemoryStream msOrig = new MemoryStream();

            await using (var ser = new CollectionSerializerAsync <SerializedItemVarLength>(msOrig, cfg))
                foreach (var item in originalArray)
                {
                    await ser.SerializeAsync(item, CancellationToken.None);
                }
            var msPB = new MemoryStream(msOrig.ToArray());

            using (var ds = new CollectionDeserializerAsync <SerializedItemVarLength>(cfg))
                await foreach (var item in ds.DeserializeAsync(msPB))
                {
                    newArray.Add(item);
                }

            Assert.AreEqual(originalArray, newArray);
        }