public FrameFormatterSerializationOptions(int totWorkerThreads, bool throwOnUnnasignedFrameDeserialization = true)
{
BatchSizeEstimatorConfig = new BatchSizeEstimatorConfig();
FIFOWorkerConfig = new FIFOWorkerConfig(totWorkerThreads);
MthWorkerConfig = new MultiThreadedWorkerConfig(totWorkerThreads);
ThrowOnUnnasignedFrameDeserialization = throwOnUnnasignedFrameDeserialization;
}
public FrameItemFormatter()
{
int defaultMaxQueuedItems = new FIFOWorkerConfig(Environment.ProcessorCount * 2).MaxQueuedItems;
objPoolBufferWriterBodies = new DefaultObjectPool <ArrayPoolBufferWriter <byte> >(
new ArrayPoolBufferWriterObjectPoolPolicy <byte>(1024 * 64), defaultMaxQueuedItems);
}
public void AddWorkItem_Cancels(int totThreads)
{
MockWorker mw = new MockWorker();
CancellationTokenSource ts = new CancellationTokenSource();
List <MockWorkOut> doneWork = new List <MockWorkOut>();
var cfg = new FIFOWorkerConfig(totThreads);
FIFOWorker <MockWorkIn, MockWorkOut> fifo = new FIFOWorker <MockWorkIn, MockWorkOut>(cfg, mw.DoMockWorkBlocking);
int countTask = fifo.AddWorkItem(new MockWorkIn(1), ts.Token).Count();
mw.TriggerOnBlockedWork(() => ts.Cancel());
Assert.Throws <TaskCanceledException>(() =>
{
try
{
fifo.Dispose();
}
catch (AggregateException ag)
{
throw ag.GetBaseException();
}
});
Assert.AreEqual(1, mw.doneWork.Count(f => f.Item1));
Assert.AreEqual(1, mw.doneWork.Count());
}
public void Dispose_CancelsPendingTasks(int totThreads)
{
TaskCompletionSource <object> taskBlocker1 = new TaskCompletionSource <object>();
TaskCompletionSource <object> taskBlocker2 = new TaskCompletionSource <object>();
bool completed = false;
int totBlockCalled = 0;
MockWorkOut DoMockWorkBlocking(MockWorkIn work, CancellationToken token)
{
Interlocked.Increment(ref totBlockCalled);
taskBlocker2.SetResult(null);
taskBlocker1.Task.Wait();
Task.Delay(1, token).Wait();
completed = true;
return(new MockWorkOut(work));
}
CancellationTokenSource ts = new CancellationTokenSource();
List <MockWorkOut> doneWork = new List <MockWorkOut>();
var cfg = new FIFOWorkerConfig(totThreads);
FIFOWorker <MockWorkIn, MockWorkOut> fifo = new FIFOWorker <MockWorkIn, MockWorkOut>(cfg, DoMockWorkBlocking);
int count = fifo.AddWorkItem(new MockWorkIn(1), ts.Token).Count();
taskBlocker2.Task.Wait();
taskBlocker1.SetResult(null);
Assert.AreEqual(1, totBlockCalled);
Assert.AreEqual(0, count);
Assert.DoesNotThrow(fifo.Dispose);
Assert.AreEqual(true, completed);
}
public void Dispose_ExceptionInWorkerPropagates(int totThreads)
{
TaskCompletionSource <object> taskBlocker1 = new TaskCompletionSource <object>();
MockWorkOut DoMockWorkBlocking(MockWorkIn work, CancellationToken token)
{
taskBlocker1.Task.Wait();
throw new MockException();
}
CancellationTokenSource ts = new CancellationTokenSource();
List <MockWorkOut> doneWork = new List <MockWorkOut>();
var cfg = new FIFOWorkerConfig(totThreads);
FIFOWorker <MockWorkIn, MockWorkOut> fifo = new FIFOWorker <MockWorkIn, MockWorkOut>(cfg, DoMockWorkBlocking);
int count = fifo.AddWorkItem(new MockWorkIn(1), ts.Token).Count();
Assert.Throws <MockException>(() =>
{
try
{
taskBlocker1.SetResult(null);
fifo.Dispose();
}
catch (AggregateException ag)
{
throw ag.GetBaseException();
}
});
}
public FrameFormatterSerializationOptions(
BatchSizeEstimatorConfig batchSizeEstimatorConfig,
MultiThreadedWorkerConfig mthWorkerConfig,
bool throwOnUnnasignedFrameDeserialization = true)
{
BatchSizeEstimatorConfig = batchSizeEstimatorConfig;
FIFOWorkerConfig = new FIFOWorkerConfig(Environment.ProcessorCount * 2);
MthWorkerConfig = mthWorkerConfig;
ThrowOnUnnasignedFrameDeserialization = throwOnUnnasignedFrameDeserialization;
}
public FrameFormatterSerializationOptions(
BatchSizeEstimatorConfig batchSizeEstimatorConfig,
FIFOWorkerConfig fIFOWorkerConfig,
MultiThreadedWorkerConfig mthWorkerConfig,
bool throwOnUnnasignedFrameDeserialization = true)
{
BatchSizeEstimatorConfig = batchSizeEstimatorConfig;
FIFOWorkerConfig = fIFOWorkerConfig;
MthWorkerConfig = mthWorkerConfig;
ThrowOnUnnasignedFrameDeserialization = throwOnUnnasignedFrameDeserialization;
}
public CollectionDeserializerAsync(FIFOWorkerConfig fifowConfig, TypeModel typeModel, int desiredBatchSize_bytes = 1024 * 64)
{
this.desiredBatchSize_bytes = desiredBatchSize_bytes;
fifow = new FIFOWorker <BatchIn, BatchOut>(fifowConfig, HandleWorkerOutput);
this.typeModel = typeModel;
t_ParallelServices_ArrayWrapper = typeof(ParallelServices_ArrayWrapper <T>);
typeModel.SetupParallelServices <T>();
objPoolBufferWriterSerializedBatch = new DefaultObjectPool <ArrayPoolBufferWriter <byte> >(
new ArrayPoolBufferWriterObjectPoolPolicy <byte>(Math.Max(1024 * 64, desiredBatchSize_bytes)),
fifowConfig.MaxQueuedItems);
objPoolList = new DefaultObjectPool <List <int> >(new ListObjectPoolPolicy <int>(64), fifowConfig.MaxQueuedItems);
}
public CollectionDeserializerAsync(FIFOWorkerConfig fifowConfig, MessagePackSerializerOptions r_opts, int desiredBatchSize_bytes = 1024 * 64)
{
fifow = new FIFOWorker <BatchWithBufferWriters, BatchWithFramesArray>(fifowConfig, HandleWorkerOutput);
this.desiredBatchSize_bytes = desiredBatchSize_bytes;
objPoolBufferWriterBodies = new DefaultObjectPool <ArrayPoolBufferWriter <byte> >(
new ArrayPoolBufferWriterObjectPoolPolicy <byte>(Math.Max(1024 * 64, desiredBatchSize_bytes)),
fifowConfig.MaxQueuedItems);
objPoolBufferWriterBodyLengths = new DefaultObjectPool <ArrayPoolBufferWriter <int> >(
new ArrayPoolBufferWriterObjectPoolPolicy <int>(1024),
fifowConfig.MaxQueuedItems);
arrPoolOutputBatch = ArrayPool <Frame <T> > .Shared;
this.r_opts = r_opts;
}
public CollectionSerializerAsync(Stream stream, FIFOWorkerConfig fifowConfig, BatchSizeEstimatorConfig estimatorConfig, TypeModel typeModel)
{
this.stream = stream;
fifow = new FIFOWorker <List <T>, MemoryStream>(fifowConfig, HandleWorkerOutput);
this.typeModel = typeModel;
typeModel.SetupParallelServices <T>();
batchEstimator = new BatchSizeEstimator(estimatorConfig);
objPoolList = new DefaultObjectPool <List <T> >(new ListObjectPoolPolicy <T>(64), fifowConfig.MaxQueuedItems);
objPoolMemoryStream = new DefaultObjectPool <MemoryStream>(
new MemoryStreamObjectPoolPolicy(Math.Max(1024 * 64, estimatorConfig.DesiredBatchSize_bytes)),
fifowConfig.MaxQueuedItems);
desiredBatchSize = 1;
currentBatch = objPoolList.Get();
}
public CollectionSerializerAsync(Stream stream, FIFOWorkerConfig fifowConfig, BatchSizeEstimatorConfig estimatorConfig, MessagePackSerializerOptions w_opts)
{
this.stream = stream;
fifow = new FIFOWorker <ArrayPoolBufferWriter <T>, BatchWithBufferWriters>(fifowConfig, HandleWorkerOutput);
batchEstimator = new BatchSizeEstimator(estimatorConfig);
this.w_opts = w_opts;
objPoolBufferWriterBodies = new DefaultObjectPool <ArrayPoolBufferWriter <byte> >(
new ArrayPoolBufferWriterObjectPoolPolicy <byte>(Math.Max(1024 * 64, estimatorConfig.DesiredBatchSize_bytes)),
fifowConfig.MaxQueuedItems);
objPoolBufferWriterBodyLengths = new DefaultObjectPool <ArrayPoolBufferWriter <int> >(
new ArrayPoolBufferWriterObjectPoolPolicy <int>(1024),
fifowConfig.MaxQueuedItems);
objPoolOutputBatch = new DefaultObjectPool <ArrayPoolBufferWriter <T> >(
new ArrayPoolBufferWriterObjectPoolPolicy <T>(1024),
fifowConfig.MaxQueuedItems);
currentBatch = objPoolOutputBatch.Get();
desiredBatchSize = 1;
formatterT = w_opts.Resolver.GetFormatterWithVerify <T>();
}
public void Dispose_ThrowsOnCancelledTasks(int totThreads)
{
TaskCompletionSource <object> taskBlocker1 = new TaskCompletionSource <object>();
TaskCompletionSource <object> taskBlocker2 = new TaskCompletionSource <object>();
bool completed = false;
int totBlockCalled = 0;
MockWorkOut DoMockWorkBlocking(MockWorkIn work, CancellationToken token)
{
Interlocked.Increment(ref totBlockCalled);
taskBlocker2.SetResult(null);
taskBlocker1.Task.Wait();
Task.Delay(1, token).Wait();
completed = true;
return(new MockWorkOut(work));
}
CancellationTokenSource ts = new CancellationTokenSource();
List <MockWorkOut> doneWork = new List <MockWorkOut>();
var cfg = new FIFOWorkerConfig(totThreads);
FIFOWorker <MockWorkIn, MockWorkOut> fifo = new FIFOWorker <MockWorkIn, MockWorkOut>(cfg, DoMockWorkBlocking);
int count = fifo.AddWorkItem(new MockWorkIn(1), ts.Token).Count();
taskBlocker2.Task.Wait();
ts.Cancel();
Assert.Throws <TaskCanceledException>(() =>
{
try
{
taskBlocker1.SetResult(null);
fifo.Dispose();
}
catch (AggregateException ag)
{
throw ag.GetBaseException();
}
});
Assert.AreEqual(1, totBlockCalled);
Assert.AreEqual(0, count);
Assert.AreEqual(false, completed);
}
public CollectionSerializerAsync(Stream stream, FIFOWorkerConfig fifowConfig, TypeModel typeModel) :
this(stream, fifowConfig, new BatchSizeEstimatorConfig(), typeModel)
{
}
public CollectionSerializerAsync(Stream stream, FIFOWorkerConfig fifowConfig) :
this(stream, fifowConfig, new BatchSizeEstimatorConfig(), RuntimeTypeModel.Default)
{
}
public CollectionDeserializerAsync(FIFOWorkerConfig fifowConfig, int desiredBatchSize_bytes = 1024 * 64) :
this(fifowConfig, RuntimeTypeModel.Default, desiredBatchSize_bytes)
{
}
public CollectionDeserializerAsync(FIFOWorkerConfig fifowConfig) :
this(fifowConfig, MessagePackSerializerOptions.Standard)
{
}
public CollectionSerializerAsync(Stream stream, FIFOWorkerConfig fifowConfig, BatchSizeEstimatorConfig estimatorConfig) :
this(stream, fifowConfig, estimatorConfig, MessagePackSerializerOptions.Standard)
{
}
