static bool HeuristicallyDetectWhetherMultithreadingMakesSense(CreateParserParams parserParams,
TextStreamPositioningParams textStreamPositioningParams)
{
#if SILVERLIGHT
return(false);
#else
if (System.Environment.ProcessorCount == 1)
{
return(false);
}
long approxBytesToRead;
if (parserParams.Direction == MessagesParserDirection.Forward)
{
approxBytesToRead = new TextStreamPosition(parserParams.Range.Value.End, textStreamPositioningParams).StreamPositionAlignedToBlockSize
- new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
}
else
{
approxBytesToRead = new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize
- new TextStreamPosition(parserParams.Range.Value.Begin, textStreamPositioningParams).StreamPositionAlignedToBlockSize;
}
if (approxBytesToRead < MultiThreadedStrategy <int> .GetBytesToParsePerThread(textStreamPositioningParams) * 2)
{
return(false);
}
return(true);
#endif
}
public IPositionedMessagesParser CreateParser(CreateParserParams parserParams)
{
parserParams.EnsureRangeIsSet(this);
var strategiesCache = new StreamParser.StrategiesCache()
{
MultiThreadedStrategy = multiThreadedStrategy,
SingleThreadedStrategy = singleThreadedStrategy
};
DejitteringParams?dejitteringParams = GetDejitteringParams();
if (dejitteringParams != null && (parserParams.Flags & MessagesParserFlag.DisableDejitter) == 0)
{
return(new DejitteringMessagesParser(
underlyingParserParams => new StreamParser(
this,
EnsureParserRangeDoesNotExceedReadersBoundaries(underlyingParserParams),
textStreamPositioningParams,
settingsAccessor,
strategiesCache
),
parserParams,
dejitteringParams.Value
));
}
return(new StreamParser(
this,
parserParams,
textStreamPositioningParams,
settingsAccessor,
strategiesCache
));
}
private CreateParserParams EnsureParserRangeDoesNotExceedReadersBoundaries(CreateParserParams p)
{
if (p.Range != null)
{
p.Range = FileRange.Range.Intersect(p.Range.Value,
new FileRange.Range(BeginPosition, EndPosition)).Common;
}
return(p);
}
public static async Task <StreamParser> Create(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache)
{
var parser = new StreamParser(owner, p, textStreamPositioningParams, globalSettings, strategiesCache);
await parser.Strategy.ParserCreated(parser.InitialParams);
return(parser);
}
async Task CreateUnderlyingParserAndInitJitterBuffer(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory)
{
CreateParserParams reversedParserParams = originalParams;
reversedParserParams.Range = null;
reversedParserParams.Direction = GetOppositeDirection(originalParams.Direction);
reversedParserParams.Flags |= MessagesParserFlag.DisableMultithreading;
int reversedMessagesQueued = 0;
await DisposableAsync.Using(await underlyingParserFactory(reversedParserParams), async reversedParser =>
{
var tmp = new List <PostprocessedMessage>();
for (int i = 0; i < jitterBufferSize; ++i)
{
var tmpMsg = await reversedParser.ReadNextAndPostprocess();
if (tmpMsg.Message == null)
{
break;
}
tmp.Add(tmpMsg);
}
tmp.Reverse();
foreach (var tmpMsg in tmp)
{
jitterBuffer.Enqueue(new Entry()
{
data = tmpMsg, index = currentIndex++
});
positionsBuffer.Push(new MessagesPositions(tmpMsg.Message));
++reversedMessagesQueued;
}
});
enumerator = await ReadAddMessagesFromRangeCompleteJitterBuffer(underlyingParserFactory).GetEnumerator();
for (int i = 0; i < jitterBufferSize; ++i)
{
var tmp = await LoadNextMessage();
reversedMessagesQueued -= tmp.DequeuedMessages;
if (tmp.LoadedMessage == null)
{
break;
}
}
for (int i = 0; i < reversedMessagesQueued && jitterBuffer.Count > 0; ++i)
{
jitterBuffer.Dequeue();
positionsBuffer.Pop();
}
}
private StreamParser(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache
)
{
p.EnsureRangeIsSet(owner);
this.InitialParams = p;
this.isSequentialReadingParser = (p.Flags & MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading) != 0;
this.multithreadingDisabled = (p.Flags & MessagesParserFlag.DisableMultithreading) != 0 ||
globalSettings.MultithreadedParsingDisabled;
CreateParsingStrategy(p, textStreamPositioningParams, strategiesCache, out this.Strategy);
}
private DejitteringMessagesParser(CreateParserParams originalParams, int jitterBufferSize)
{
if (jitterBufferSize < 1)
{
throw new ArgumentException("jitterBufferSize must be equal to or geater than 1");
}
if (originalParams.Range == null)
{
throw new ArgumentNullException("DejitteringMessagesParser does not support unspecified positions range", "originalParams.Range");
}
this.originalParams = originalParams;
this.originalParams.EnsureStartPositionIsInRange();
this.jitterBufferSize = jitterBufferSize;
this.jitterBuffer = new VCSKicksCollection.PriorityQueue <Entry>(new Comparer(originalParams.Direction, jitterBufferSize));
this.positionsBuffer = new Generic.CircularBuffer <MessagesPositions>(jitterBufferSize + 1);
}
IEnumerableAsync <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer(
Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory)
{
return(EnumerableAsync.Produce <PostprocessedMessage>(async yieldAsync =>
{
CreateParserParams mainParserParams = originalParams;
//mainParserParams.Range = null;
await DisposableAsync.Using(await underlyingParserFactory(mainParserParams), async mainParser =>
{
for (; ;)
{
var msg = await mainParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
if (!await yieldAsync.YieldAsync(msg))
{
break;
}
}
});
CreateParserParams jitterBufferCompletionParams = originalParams;
jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading;
jitterBufferCompletionParams.Range = null;
jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin;
await DisposableAsync.Using(await underlyingParserFactory(jitterBufferCompletionParams), async completionParser =>
{
for (int i = 0; i < jitterBufferSize; ++i)
{
var msg = await completionParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
if (!await yieldAsync.YieldAsync(msg))
{
break;
}
}
});
}));
}
void CreateParsingStrategy(
CreateParserParams parserParams,
TextStreamPositioningParams textStreamPositioningParams,
StrategiesCache strategiesCache,
out BaseStrategy strategy)
{
bool useMultithreadedStrategy;
if (multithreadingDisabled)
{
useMultithreadedStrategy = false;
}
else if (!isSequentialReadingParser)
{
useMultithreadedStrategy = false;
}
else
{
useMultithreadedStrategy = HeuristicallyDetectWhetherMultithreadingMakesSense(parserParams, textStreamPositioningParams);
}
//useMultithreadedStrategy = false;
Lazy <BaseStrategy> strategyToTryFirst;
Lazy <BaseStrategy> strategyToTrySecond;
if (useMultithreadedStrategy)
{
strategyToTryFirst = strategiesCache.MultiThreadedStrategy;
strategyToTrySecond = strategiesCache.SingleThreadedStrategy;
}
else
{
strategyToTryFirst = strategiesCache.SingleThreadedStrategy;
strategyToTrySecond = strategiesCache.MultiThreadedStrategy;
}
strategy = strategyToTryFirst.Value;
if (strategy == null)
{
strategy = strategyToTrySecond.Value;
}
}
public static async Task <DejitteringMessagesParser> Create(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory,
CreateParserParams originalParams, int jitterBufferSize)
{
if (underlyingParserFactory == null)
{
throw new ArgumentNullException("underlyingParserFactory");
}
var parser = new DejitteringMessagesParser(originalParams, jitterBufferSize);
try
{
await parser.CreateUnderlyingParserAndInitJitterBuffer(underlyingParserFactory);
}
catch
{
await parser.Dispose();
throw;
}
return(parser);
}
IEnumerable <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer(Func <CreateParserParams, IPositionedMessagesParser> underlyingParserFactory)
{
CreateParserParams mainParserParams = originalParams;
//mainParserParams.Range = null;
using (var mainParser = underlyingParserFactory(mainParserParams))
{
for (; ;)
{
var msg = mainParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
yield return(msg);
}
}
CreateParserParams jitterBufferCompletionParams = originalParams;
jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading;
jitterBufferCompletionParams.Range = null;
jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin;
using (var completionParser = underlyingParserFactory(jitterBufferCompletionParams))
{
for (int i = 0; i < jitterBufferSize; ++i)
{
var msg = completionParser.ReadNextAndPostprocess();
if (msg.Message == null)
{
break;
}
yield return(msg);
}
}
}
public IPositionedMessagesParser CreateParser(CreateParserParams p)
{
return(underliyingReader.CreateParser(p));
}
public DejitteringMessagesParser(Func <CreateParserParams, IPositionedMessagesParser> underlyingParserFactory,
CreateParserParams originalParams, DejitteringParams config) :
this(underlyingParserFactory, originalParams, config.JitterBufferSize)
{
}
public static Task <DejitteringMessagesParser> Create(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory,
CreateParserParams originalParams, DejitteringParams config)
{
return(Create(underlyingParserFactory, originalParams, config.JitterBufferSize));
}
